Disable harvesting of JVM metrics via ELASTICSEARCHATTRIBUTE=disabled for netvirt... 56/85756/11
authorStanislav Chlebec <stanislav.chlebec@pantheon.tech>
Wed, 13 Nov 2019 15:33:39 +0000 (16:33 +0100)
committerStanislav Chlebec <stanislav.chlebec@pantheon.tech>
Fri, 15 Nov 2019 11:49:52 +0000 (12:49 +0100)
The files set_elasticsearch_attribute.sh and set_jvm_common_attribute.sh which were removed from the repo integration/test were transformed into functions in jjb/integration/common-functions.sh

Minor fix - fixed Connection refused message.
...
unix  3      [ ]         STREAM     CONNECTED     18101
ssh to karaf console to do bundle refresh of decanter jmx collector
ssh: connect to host localhost port 8101: Connection refused
...

JIRA: INTTEST-63
Signed-off-by: Stanislav Chlebec <stanislav.chlebec@pantheon.tech>
Change-Id: If710cd4b19a988e941ebac2b0f129dbe7e15d473

jjb/integration/common-functions.sh
jjb/integration/integration-macros.yaml
jjb/netvirt/netvirt-apex-images.yaml
jjb/netvirt/netvirt-csit-hwvtep-openstack.yaml
jjb/netvirt/netvirt-csit-multi-apex.yaml
jjb/netvirt/netvirt-csit-multi-openstack-upgrade.yaml
jjb/netvirt/netvirt-csit-multi-openstack.yaml
jjb/netvirt/netvirt-dependencies.yaml
jjb/netvirt/netvirt-patch-test.yaml
jjb/netvirt/netvirt.yaml

index 143b830fefe258430e3d768e9124f06c2123a302..33b3f64c6f9b01f0cabf7afed10a13097eecafc3 100644 (file)
@@ -223,9 +223,10 @@ function get_test_suites() {
         testplan_filepath="${WORKSPACE}/test/csit/testplans/${TESTPLAN}"
     fi
 
-    add_test="integration/test/csit/suites/integration/Create_JVM_Plots.robot" # we should always add for preparing JVM monitoring
-    echo >> "$testplan_filepath"
-    echo "${add_test}" >> "$testplan_filepath"
+    if [ "${ELASTICSEARCHATTRIBUTE}" != "disabled" ]; then
+        add_test="integration/test/csit/suites/integration/Create_JVM_Plots.robot"
+        echo "${add_test}" >> "$testplan_filepath"
+    fi
 
     echo "Changing the testplan path..."
     sed "s:integration:${WORKSPACE}:" "${testplan_filepath}" > testplan.txt
@@ -289,22 +290,147 @@ function run_plan() {
 # Run scripts to support JVM monitoring.
 function add_jvm_support()
 {
-    # TODO unite short and long version to one script and parametrize the input: short/long/any number
-    if [ "${ELASTICSEARCHATTRIBUTE}" == "short" ]; then
-        run_script="${WORKSPACE}/test/csit/scripts/set_elasticsearch_attribute_short.sh"
-    else
-        run_script="${WORKSPACE}/test/csit/scripts/set_elasticsearch_attribute_long.sh"
+    if [ "${ELASTICSEARCHATTRIBUTE}" != "disabled" ]; then
+        set_elasticsearch_attribute "${ELASTICSEARCHATTRIBUTE}"
+        #run_script="${WORKSPACE}/test/csit/scripts/set_elasticsearch_attribute.sh ${ELASTICSEARCHATTRIBUTE}"
+        #printf "Executing %s...\\n" "${run_script}"
+        ## shellcheck source=${line} disable=SC1091
+        #source "${run_script}"
+
+        set_jvm_common_attribute
+        #run_script="${WORKSPACE}/test/csit/scripts/set_jvm_common_attribute.sh"
+        #printf "Executing %s...\\n" "${run_script}"
+        ## shellcheck source=${line} disable=SC1091
+        #source "${run_script}"
     fi
-    printf "Executing %s...\\n" "${run_script}"
-    # shellcheck source=${line} disable=SC1091
-    source "${run_script}"
-
-    run_script="${WORKSPACE}/test/csit/scripts/set_jvm_common_attribute.sh"
-    printf "Executing %s...\\n" "${run_script}"
-    # shellcheck source=${line} disable=SC1091
-    source "${run_script}"
 } # function add_jvm_support()
 
+#Expected input parameter: long/short/a number
+function set_elasticsearch_attribute()
+{
+short=5000
+long=120000
+default=$short
+
+case $1 in
+short)
+  period=$short
+  ;;
+long)
+  period=$long
+  ;;
+*)
+  # shellcheck disable=SC2166
+  if [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -ge $short -a "$1" -le $long ]; then
+      period=$1
+  else
+      period=$default
+  fi
+  ;;
+esac
+
+cat > "${WORKSPACE}"/org.apache.karaf.decanter.scheduler.simple.cfg <<EOF
+period=$period
+
+EOF
+
+echo "Copying config files to ODL Controller folder"
+
+# shellcheck disable=SC2086
+for i in $(seq 1 ${NUM_ODL_SYSTEM})
+do
+        CONTROLLERIP=ODL_SYSTEM_${i}_IP
+        echo "Setup long duration config to ${!CONTROLLERIP}"
+        # We purposely want these variables to expand client-side
+        # shellcheck disable=SC2029
+        ssh "${!CONTROLLERIP}" "mkdir -p /tmp/${BUNDLEFOLDER}/etc/opendaylight/karaf/"
+        scp "${WORKSPACE}"/org.apache.karaf.decanter.scheduler.simple.cfg "${!CONTROLLERIP}":/tmp/"${BUNDLEFOLDER}"/etc/
+done
+} #function set_elasticsearch_attribute
+
+function set_jvm_common_attribute()
+{
+cat > "${WORKSPACE}"/org.apache.karaf.decanter.collector.jmx-local.cfg <<EOF
+type=jmx-local
+url=local
+object.name=java.lang:type=*,name=*
+
+EOF
+
+cat > "${WORKSPACE}"/org.apache.karaf.decanter.collector.jmx-others.cfg <<EOF
+type=jmx-local
+url=local
+object.name=java.lang:type=*
+
+EOF
+
+# shellcheck disable=SC2086
+for i in $(seq 1 ${NUM_ODL_SYSTEM})
+do
+    CONTROLLERIP=ODL_SYSTEM_${i}_IP
+    CLUSTERNAME=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 12)
+
+    cat > "${WORKSPACE}"/elasticsearch.yml <<EOF
+    cluster.name: ${CLUSTERNAME}
+    network.host: ${!CONTROLLERIP}
+    discovery.zen.ping.multicast.enabled: false
+
+EOF
+    cat > "${WORKSPACE}"/org.apache.karaf.decanter.appender.elasticsearch.cfg <<EOF
+    host=${!CONTROLLERIP}
+    port=9300
+    clusterName=${CLUSTERNAME}
+
+EOF
+
+    cat > "${WORKSPACE}"/elasticsearch_startup.sh <<EOF
+    cd /tmp/elasticsearch/elasticsearch-1.7.5
+    ls -al
+
+    if [ -d "data" ]; then
+        echo "data directory exists, deleting...."
+        rm -r data
+    else
+        echo "data directory does not exist"
+    fi
+
+    cd /tmp/elasticsearch
+    ls -al
+
+    echo "Starting Elasticsearch node"
+    sudo /tmp/elasticsearch/elasticsearch-1.7.5/bin/elasticsearch > /dev/null 2>&1 &
+    ls -al /tmp/elasticsearch/elasticsearch-1.7.5/bin/elasticsearch
+
+EOF
+    echo "Setup ODL_SYSTEM_IP specific config files for ${!CONTROLLERIP} "
+
+    cat "${WORKSPACE}"/org.apache.karaf.decanter.appender.elasticsearch.cfg
+    cat "${WORKSPACE}"/org.apache.karaf.decanter.collector.jmx-local.cfg
+    cat "${WORKSPACE}"/org.apache.karaf.decanter.collector.jmx-others.cfg
+    cat "${WORKSPACE}"/elasticsearch.yml
+
+
+    echo "Copying config files to ${!CONTROLLERIP}"
+
+    scp "${WORKSPACE}"/org.apache.karaf.decanter.appender.elasticsearch.cfg "${!CONTROLLERIP}":/tmp/"${BUNDLEFOLDER}"/etc/
+    scp "${WORKSPACE}"/org.apache.karaf.decanter.collector.jmx-local.cfg "${!CONTROLLERIP}":/tmp/"${BUNDLEFOLDER}"/etc/
+    scp "${WORKSPACE}"/org.apache.karaf.decanter.collector.jmx-others.cfg "${!CONTROLLERIP}":/tmp/"${BUNDLEFOLDER}"/etc/
+
+    scp "${WORKSPACE}"/elasticsearch.yml "${!CONTROLLERIP}":/tmp/
+
+    ssh "${!CONTROLLERIP}" "sudo ls -al /tmp/elasticsearch/"
+
+    ssh "${!CONTROLLERIP}" "sudo mv /tmp/elasticsearch.yml /tmp/elasticsearch/elasticsearch-1.7.5/config/"
+    ssh "${!CONTROLLERIP}" "cat /tmp/elasticsearch/elasticsearch-1.7.5/config/elasticsearch.yml"
+
+    echo "Copying the elasticsearch_startup script to ${!CONTROLLERIP}"
+    cat "${WORKSPACE}"/elasticsearch_startup.sh
+    scp "${WORKSPACE}"/elasticsearch_startup.sh "${!CONTROLLERIP}":/tmp
+    ssh "${!CONTROLLERIP}" 'bash /tmp/elasticsearch_startup.sh'
+    ssh "${!CONTROLLERIP}" 'ps aux | grep elasticsearch'
+done
+} #function set_jvm_common_attribute
+
 # Return elapsed time. Usage:
 # - Call first time with no arguments and a new timer is returned.
 # - Next call with the first argument as the timer and the elapsed time is returned.
@@ -761,8 +887,10 @@ function get_features() {
         ACTUALFEATURES="odl-infrautils-ready,${CONTROLLERFEATURES}"
     fi
 
-    # Add decanter features to allow JVM monitoring
-    ACTUALFEATURES="${ACTUALFEATURES},decanter-collector-jmx,decanter-appender-elasticsearch"
+    if [ "${ELASTICSEARCHATTRIBUTE}" != "disabled" ]; then
+        # Add decanter features to allow JVM monitoring
+        ACTUALFEATURES="${ACTUALFEATURES},decanter-collector-jmx,decanter-appender-elasticsearch"
+    fi
 
     # Some versions of jenkins job builder result in feature list containing spaces
     # and ending in newline. Remove all that.
@@ -861,7 +989,7 @@ function create_post_startup_script() {
 # wait up to 60s for karaf port 8101 to be opened, polling every 5s
 loop_count=0;
 until [[ \$loop_count -ge 12 ]]; do
-    netstat -na | grep 8101 && break;
+    netstat -na | grep ":::8101" && break;
     loop_count=\$[\$loop_count+1];
     sleep 5;
 done
index c3e90c3cd3f6e532719ce0ec52d6d78247cde587..6768e4ce61d7183abc2b0841cf4a3e475e64c9bb 100644 (file)
@@ -38,7 +38,8 @@
           default: '{controller-features}'
           description: |
               Features to install in the controller separated by comma
-              (note: extra features such as decanter are inserted in install scripts)
+              (note: Do not include decanter* features used for harvesting JVM metrics.
+               They are inserted in install scripts.)
 
       - string:
           name: CONTROLLERDEBUGMAP
           default: '{feature-elasticsearch-attribute}'
           description: |
               Configure the period the script should harvest JVM metrics.
-              Default is short.
-              Possible values: long (its value is 120000)
-                               short (its value is 5000)
+              Possible values: long (=120000)
+                               short (=5000)
                                any number between 5000 and 120000
+                               disabled (use for disabling harvesting)
 
       - string:
           name: USEFEATURESBOOT
index 4b9ebd50e416f43030eb48bd75b5641458873873..e484a9484a1193044c17fd4cc0da279c7ee90b7c 100644 (file)
@@ -5,6 +5,7 @@
       - 'netvirt-apex-image-management'
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     os-cloud: vex
     disable-job: true
 
index 4d98bd450a529402e5b8991433bf4f2dbca5d29b..36e2626ffb379831c88b25f1a037161e0945dc66 100644 (file)
@@ -5,6 +5,7 @@
       - inttest-csit-hwvtep-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream: 'magnesium'
     branch: 'master'
     jre: 'openjdk11'
@@ -55,6 +56,7 @@
       - inttest-csit-hwvtep-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream: 'sodium'
     branch: 'stable/sodium'
     testplan: '{project}-{topology}-l2gw-openstack.txt'
       - inttest-csit-hwvtep-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream: 'neon'
     branch: 'stable/neon'
     testplan: '{project}-{topology}-l2gw-openstack.txt'
index 3b23bbf1aea954537a717ca7b89c3d3c3d7c6f52..542bb4e481dffa26309018cadbadc7d79a43f32b 100644 (file)
@@ -7,6 +7,7 @@
     disable-job: true
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream:
       - magnesium:
           branch: 'master'
index 4ed7f4ff1c340b45cd2f8955b4c673f09fb80487..e0da860e9603820a457b02894a74ff8b8f85bdca 100644 (file)
@@ -5,6 +5,7 @@
       - inttest-csit-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
 
     stream:
       - magnesium:
index 84a1f5ed181e794befea8685ba7acf123a143fc8..904d3b90b63f90b34f3c3faf9543ee3493949530 100644 (file)
@@ -5,6 +5,7 @@
       - inttest-csit-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream: 'magnesium'
     branch: 'master'
     jre: 'openjdk11'
@@ -94,6 +95,7 @@
       - inttest-csit-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream: 'sodium'
     branch: 'stable/sodium'
     testplan: '{project}-{topology}-openstack.txt'
       - inttest-csit-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     stream: 'neon'
     branch: 'stable/neon'
     testplan: '{project}-{topology}-openstack.txt'
       - inttest-csit-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     security-group-mode: 'stateful'
     odl_system_flavor: odl-highcpu-4
     openstack_system2_flavor: odl-highcpu-8
       - inttest-csit-openstack-custom
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     testplan: '{project}-{topology}-openstack.txt'
     odl-ml2-driver-version: 'v2'
     enable-openstack-services: 'n-cpu,placement-api,tempest'
index db1b70c2c417c3de0094fed95efc7880bf7a4fb2..7d8e9e021aaa05b019270f47f86a461c46ceaa97 100644 (file)
@@ -3,6 +3,7 @@
     name: netvirt-thirdparty-quagga-dependencies
     project: 'netvirt'
     project-name: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     jobs:
       - 'netvirt-thirdparty-quagga-{platform}'
 
index e141afbc2c1a344d53f28104e239a8b9ed86db0f..a28c376039c4e6795d8883ac2f25acefbce9bc48 100644 (file)
@@ -5,6 +5,7 @@
       - inttest-patch-test
 
     project: 'netvirt'
+    elasticsearch-attribute: 'disabled'
 
     stream:
       - magnesium:
index 03c1962adc7370193f10f7e405f09c5f63117c22..76b32bc545bf1ebcc151f448e8ac969b02166a5e 100644 (file)
@@ -22,6 +22,7 @@
 
     project: 'netvirt'
     project-name: 'netvirt'
+    elasticsearch-attribute: 'disabled'
 
     java-version: 'openjdk11'
     mvn-settings: 'netvirt-settings'
@@ -63,6 +64,7 @@
 
     project: 'netvirt'
     project-name: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     mvn-settings: 'netvirt-settings'
     mvn-opts: '-Xmx2048m'
     build-timeout: 90
 
     project: 'netvirt'
     project-name: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     mvn-settings: 'netvirt-settings'
     mvn-opts: '-Xmx2048m'
     build-timeout: 90
 
     project: 'netvirt'
     project-name: 'netvirt'
+    elasticsearch-attribute: 'disabled'
     branch: 'master'
     java-version: 'openjdk11'
     mvn-settings: 'netvirt-settings'