From 06e0024cee0d193020ad2b5dfd0eea49ab2fcbeb Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Wed, 30 Oct 2019 17:19:58 -0700 Subject: [PATCH] Make cluster script use common functions Cluster script should use common functions vs custom commands Also repurpose and rename create_controller_variables function to dump_controller_threads. Change-Id: Ic60a2d104afeed8060a2ad42fa15e0b308d4c7a3 Signed-off-by: Luis Gomez --- jjb/integration/common-functions.sh | 4 +- .../integration-deploy-controller-run-test.sh | 10 +-- .../integration-start-cluster-run-test.sh | 76 +------------------ 3 files changed, 6 insertions(+), 84 deletions(-) diff --git a/jjb/integration/common-functions.sh b/jjb/integration/common-functions.sh index e38b07ae0..ccea7fcf1 100644 --- a/jjb/integration/common-functions.sh +++ b/jjb/integration/common-functions.sh @@ -950,11 +950,9 @@ function copy_and_run_post_startup_script() { done } -function create_controller_variables() { - echo "Generating controller variables..." +function dump_controller_threads() { for i in $(seq 1 "${NUM_ODL_SYSTEM}"); do CONTROLLERIP="ODL_SYSTEM_${i}_IP" - odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}" echo "Lets's take the karaf thread dump" ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}"/ps_before.log pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}"/ps_before.log | grep -v grep | tr -s ' ' | cut -f2 -d' ') diff --git a/jjb/integration/integration-deploy-controller-run-test.sh b/jjb/integration/integration-deploy-controller-run-test.sh index 3a274190d..efb108779 100755 --- a/jjb/integration/integration-deploy-controller-run-test.sh +++ b/jjb/integration/integration-deploy-controller-run-test.sh @@ -32,7 +32,7 @@ copy_and_run_startup_script copy_and_run_post_startup_script -create_controller_variables +dump_controller_threads if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then echo "Exiting without running tests to deploy openstack for testing" @@ -78,13 +78,7 @@ robot -N "${TESTPLAN}" \ -v ODL_SYSTEM_IP:"${ODL_SYSTEM_IP}" \ -v ODL_SYSTEM_USER:"${USER}" \ -v SUITES:"${SUITES}" \ - -v TOOLS_SYSTEM_1_IP:"${TOOLS_SYSTEM_IP}" \ - -v TOOLS_SYSTEM_2_IP:"${TOOLS_SYSTEM_2_IP}" \ - -v TOOLS_SYSTEM_3_IP:"${TOOLS_SYSTEM_3_IP}" \ - -v TOOLS_SYSTEM_4_IP:"${TOOLS_SYSTEM_4_IP}" \ - -v TOOLS_SYSTEM_5_IP:"${TOOLS_SYSTEM_5_IP}" \ - -v TOOLS_SYSTEM_6_IP:"${TOOLS_SYSTEM_6_IP}" \ - -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" \ + -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" ${tools_variables} \ -v TOOLS_SYSTEM_USER:"${USER}" \ -v USER_HOME:"${HOME}" \ -v WORKSPACE:/tmp \ diff --git a/jjb/integration/integration-start-cluster-run-test.sh b/jjb/integration/integration-start-cluster-run-test.sh index 16c1eb971..368e151bf 100644 --- a/jjb/integration/integration-start-cluster-run-test.sh +++ b/jjb/integration/integration-start-cluster-run-test.sh @@ -13,92 +13,22 @@ echo "#################################################" echo "## Verify Cluster is UP ##" echo "#################################################" -cat > "${WORKSPACE}/verify-cluster-is-up.sh" < "600" )); then - echo Timeout Controller DOWN - echo "Dumping first 500K bytes of karaf log..." - head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Dumping last 500K bytes of karaf log..." - tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Listing all open ports on controller system" - netstat -pnatu - exit 1 - else - COUNT=\$(( \${COUNT} + 1 )) - sleep 1 - if [[ \$((\$COUNT % 5)) == 0 ]]; then - echo already waited \${COUNT} seconds... - fi - fi -done - -echo "Listing all open ports on controller system.." -netstat -pnatu +create_post_startup_script -function exit_on_log_file_message { - echo "looking for \"\$1\" in log file" - if grep --quiet "\$1" "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"; then - echo ABORTING: found "\$1" - echo "Dumping first 500K bytes of karaf log..." - head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Dumping last 500K bytes of karaf log..." - tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" - exit 1 - fi -} +copy_and_run_post_startup_script -exit_on_log_file_message 'BindException: Address already in use' -exit_on_log_file_message 'server is unhealthy' - -EOF - -for i in $(seq 1 "${NUM_ODL_SYSTEM}") -do - CONTROLLERIP=ODL_SYSTEM_${i}_IP - echo "Verifying member-${i} with IP address ${!CONTROLLERIP} is UP" - scp "${WORKSPACE}/verify-cluster-is-up.sh" "${!CONTROLLERIP}:/tmp" - # shellcheck disable=SC2029 - ssh "${!CONTROLLERIP}" "bash /tmp/verify-cluster-is-up.sh ${i} ${!CONTROLLERIP}" -done +dump_controller_threads if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then echo "Exiting without running tests to deploy openstack for testing" exit fi -if [ "${CONTROLLERSCOPE}" == 'all' ]; then - COOLDOWN_PERIOD="180" -else - COOLDOWN_PERIOD="60" -fi -echo "Cool down for ${COOLDOWN_PERIOD} seconds :)..." -sleep "${COOLDOWN_PERIOD}" - echo "Generating controller variables..." for i in $(seq 1 "${NUM_ODL_SYSTEM}") do CONTROLLERIP=ODL_SYSTEM_${i}_IP odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}" - echo "Lets's take the karaf thread dump" - ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_before.log" - pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_before.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ') - echo "karaf main: org.apache.karaf.main.Main, pid:${pid}" - # shellcheck disable=SC2029 - ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_before.log" || true done echo "Generating mininet variables..." -- 2.36.6