Merge "Remove SM distribution jobs"

[releng/builder.git] / jjb / integration / integration-start-cluster-run-test.sh

diff --git a/jjb/integration/integration-start-cluster-run-test.sh b/jjb/integration/integration-start-cluster-run-test.sh
index 2f2b14c1d84fd3f7315f09ace27a341007e7fe65..81c30c927114b60789466e1b6f18cdce98848e31 100644 (file)
--- a/jjb/integration/integration-start-cluster-run-test.sh
+++ b/jjb/integration/integration-start-cluster-run-test.sh
@@ -4,7 +4,7 @@
 # ${ROBOT_VENV} comes from the integration-install-robotframework.sh
 # script.
 # shellcheck source=${ROBOT_VENV}/bin/activate disable=SC1091
-source "${ROBOT_VENV}/bin/activate"
+. "${ROBOT_VENV}/bin/activate"
 source /tmp/common-functions.sh "${BUNDLEFOLDER}"
 # Ensure we fail the job if any steps fail.
 set -ex -o pipefail
@@ -13,92 +13,22 @@ echo "#################################################"
 echo "##         Verify Cluster is UP                ##"
 echo "#################################################"
 
-cat > "${WORKSPACE}/verify-cluster-is-up.sh" <<EOF
-
-CONTROLLERID="member-\$1"
-ODL_SYSTEM_IP_PATH=\$2
-
-echo "Waiting for controller to come up..."
-COUNT="0"
-while true; do
-    RESP="\$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null )"
-    echo \$RESP
-    SHARD="\$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/jolokia/read/org.opendaylight.controller:Category=Shards,name=\$CONTROLLERID-shard-inventory-config,type=DistributedConfigDatastore)"
-    echo \$SHARD
-    if ([[ \$RESP == *"200"* ]] && [[ \$SHARD  == *'"status":200'* ]]); then
-        echo Controller is UP
-        break
-    elif (( "\$COUNT" > "600" )); then
-        echo Timeout Controller DOWN
-        echo "Dumping first 500K bytes of karaf log..."
-        head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Dumping last 500K bytes of karaf log..."
-        tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Listing all open ports on controller system"
-        netstat -pnatu
-        exit 1
-    else
-        COUNT=\$(( \${COUNT} + 1 ))
-        sleep 1
-        if [[ \$((\$COUNT % 5)) == 0 ]]; then
-            echo already waited \${COUNT} seconds...
-        fi
-    fi
-done
-
-echo "Listing all open ports on controller system.."
-netstat -pnatu
+create_post_startup_script
 
-function exit_on_log_file_message {
-    echo "looking for \"\$1\" in log file"
-    if grep --quiet "\$1" "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"; then
-        echo ABORTING: found "\$1"
-        echo "Dumping first 500K bytes of karaf log..."
-        head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Dumping last 500K bytes of karaf log..."
-        tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
-        exit 1
-    fi
-}
+copy_and_run_post_startup_script
 
-exit_on_log_file_message 'BindException: Address already in use'
-exit_on_log_file_message 'server is unhealthy'
-
-EOF
-
-for i in $(seq 1 "${NUM_ODL_SYSTEM}")
-do
-    CONTROLLERIP=ODL_SYSTEM_${i}_IP
-    echo "Verifying member-${i} with IP address ${!CONTROLLERIP} is UP"
-    scp "${WORKSPACE}/verify-cluster-is-up.sh" "${!CONTROLLERIP}:/tmp"
-    # shellcheck disable=SC2029
-    ssh "${!CONTROLLERIP}" "bash /tmp/verify-cluster-is-up.sh ${i} ${!CONTROLLERIP}"
-done
+dump_controller_threads
 
 if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then
    echo "Exiting without running tests to deploy openstack for testing"
    exit
 fi
 
-if [ "${CONTROLLERSCOPE}" == 'all' ]; then
-    COOLDOWN_PERIOD="180"
-else
-    COOLDOWN_PERIOD="60"
-fi
-echo "Cool down for ${COOLDOWN_PERIOD} seconds :)..."
-sleep "${COOLDOWN_PERIOD}"
-
 echo "Generating controller variables..."
 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
 do
     CONTROLLERIP=ODL_SYSTEM_${i}_IP
     odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}"
-    echo "Lets's take the karaf thread dump"
-    ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_before.log"
-    pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_before.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
-    echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"
-    # shellcheck disable=SC2029
-    ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_before.log" || true
 done
 
 echo "Generating mininet variables..."
@@ -111,8 +41,10 @@ done
 get_test_suites SUITES
 
 echo "Starting Robot test suites ${SUITES} ..."
+# ${TESTOPTIONS}, ${SUITES} are space-separated parameters and should not be quoted.
+# shellcheck disable=SC2086
 robot -N "${TESTPLAN}" \
-      --removekeywords wuks -c critical -e exclude -e "skip_if_${DISTROSTREAM}" \
+      --removekeywords wuks -e exclude -e "skip_if_${DISTROSTREAM}" \
       -v BUNDLEFOLDER:"${BUNDLEFOLDER}" \
       -v BUNDLE_URL:"${ACTUAL_BUNDLE_URL}" \
       -v CONTROLLER:"${ODL_SYSTEM_IP}" \
@@ -130,13 +62,14 @@ robot -N "${TESTPLAN}" \
       -v NUM_ODL_SYSTEM:"${NUM_ODL_SYSTEM}" \
       -v NUM_TOOLS_SYSTEM:"${NUM_TOOLS_SYSTEM}" \
       -v ODL_STREAM:"${DISTROSTREAM}" \
-      -v ODL_SYSTEM_IP:"${ODL_SYSTEM_IP}" "${odl_variables}" \
+      -v ODL_SYSTEM_IP:"${ODL_SYSTEM_IP}" ${odl_variables} \
       -v ODL_SYSTEM_USER:"${USER}" \
-      -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" "${tools_variables}" \
+      -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" ${tools_variables} \
       -v TOOLS_SYSTEM_USER:"${USER}" \
       -v USER_HOME:"${HOME}" \
+      -v IS_KARAF_APPL:"${IS_KARAF_APPL}" \
       -v WORKSPACE:/tmp \
-      "${TESTOPTIONS}" "${SUITES}" || true
+      ${TESTOPTIONS} ${SUITES} || true
 
 
 
@@ -158,7 +91,7 @@ set +e  # We do not want to create red dot just because something went wrong whi
 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
 do
     CONTROLLERIP="ODL_SYSTEM_${i}_IP"
-    echo "Lets's take the karaf thread dump again"
+    echo "Let's take the karaf thread dump again"
     ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_after.log"
     pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_after.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
     echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"