2 #@IgnoreInspection BashAddShebang
3 # Activate robotframework virtualenv
4 # ${ROBOT_VENV} comes from the integration-install-robotframework.sh
6 # shellcheck source=${ROBOT_VENV}/bin/activate disable=SC1091
7 source "${ROBOT_VENV}/bin/activate"
8 source /tmp/common-functions.sh "${BUNDLEFOLDER}"
9 # Ensure we fail the job if any steps fail.
12 echo "#################################################"
13 echo "## Verify Cluster is UP ##"
14 echo "#################################################"
16 create_post_startup_script
18 copy_and_run_post_startup_script
20 dump_controller_threads
22 if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then
23 echo "Exiting without running tests to deploy openstack for testing"
27 echo "Generating controller variables..."
28 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
30 CONTROLLERIP=ODL_SYSTEM_${i}_IP
31 odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}"
34 echo "Generating mininet variables..."
35 for i in $(seq 1 "${NUM_TOOLS_SYSTEM}")
37 MININETIP=TOOLS_SYSTEM_${i}_IP
38 tools_variables=${tools_variables}" -v ${MININETIP}:${!MININETIP}"
41 get_test_suites SUITES
43 echo "Starting Robot test suites ${SUITES} ..."
44 # ${TESTOPTIONS}, ${SUITES} are space-separated parameters and should not be quoted.
45 # shellcheck disable=SC2086
46 robot -N "${TESTPLAN}" \
47 --removekeywords wuks -c critical -e exclude -e "skip_if_${DISTROSTREAM}" \
48 -v BUNDLEFOLDER:"${BUNDLEFOLDER}" \
49 -v BUNDLE_URL:"${ACTUAL_BUNDLE_URL}" \
50 -v CONTROLLER:"${ODL_SYSTEM_IP}" \
51 -v CONTROLLER1:"${ODL_SYSTEM_2_IP}" \
52 -v CONTROLLER2:"${ODL_SYSTEM_3_IP}" \
53 -v CONTROLLER_USER:"${USER}" \
54 -v JAVA_HOME:"${JAVA_HOME}" \
55 -v JDKVERSION:"${JDKVERSION}" \
56 -v JENKINS_WORKSPACE:"${WORKSPACE}" \
57 -v MININET:"${TOOLS_SYSTEM_IP}" \
58 -v MININET1:"${TOOLS_SYSTEM_2_IP}" \
59 -v MININET2:"${TOOLS_SYSTEM_3_IP}" \
60 -v MININET_USER:"${USER}" \
61 -v NEXUSURL_PREFIX:"${NEXUSURL_PREFIX}" \
62 -v NUM_ODL_SYSTEM:"${NUM_ODL_SYSTEM}" \
63 -v NUM_TOOLS_SYSTEM:"${NUM_TOOLS_SYSTEM}" \
64 -v ODL_STREAM:"${DISTROSTREAM}" \
65 -v ODL_SYSTEM_IP:"${ODL_SYSTEM_IP}" ${odl_variables} \
66 -v ODL_SYSTEM_USER:"${USER}" \
67 -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" ${tools_variables} \
68 -v TOOLS_SYSTEM_USER:"${USER}" \
69 -v USER_HOME:"${HOME}" \
70 -v IS_KARAF_APPL:"{IS_KARAF_APPL}" \
72 ${TESTOPTIONS} ${SUITES} || true
76 echo "Examining the files in data/log and checking filesize"
77 # shellcheck disable=SC2029
78 ssh "${ODL_SYSTEM_1_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
79 # shellcheck disable=SC2029
80 ssh "${ODL_SYSTEM_1_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
81 # shellcheck disable=SC2029
82 ssh "${ODL_SYSTEM_2_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
83 # shellcheck disable=SC2029
84 ssh "${ODL_SYSTEM_2_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
85 # shellcheck disable=SC2029
86 ssh "${ODL_SYSTEM_3_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
87 # shellcheck disable=SC2029
88 ssh "${ODL_SYSTEM_3_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
90 set +e # We do not want to create red dot just because something went wrong while fetching logs.
91 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
93 CONTROLLERIP="ODL_SYSTEM_${i}_IP"
94 echo "Let's take the karaf thread dump again"
95 ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_after.log"
96 pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_after.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
97 echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"
98 # shellcheck disable=SC2029
99 ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_after.log" || true
100 echo "killing karaf process..."
101 ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
104 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
106 CONTROLLERIP=ODL_SYSTEM_${i}_IP
107 echo "Compressing karaf.log ${i}"
108 ssh "${!CONTROLLERIP}" gzip --best "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
109 echo "Fetching compressed karaf.log ${i}"
110 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.gz" "odl${i}_karaf.log.gz" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/karaf.log.gz"
111 # TODO: Should we compress the output log file as well?
112 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf_console.log" "odl${i}_karaf_console.log" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/karaf_console.log"
114 # FIXME: Put member index in filename, instead of directory name.
115 mkdir -p "gclogs-${i}"
116 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/*.log" "gclogs-${i}/" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/*.log"
119 echo "Examine copied files"
122 true # perhaps Jenkins is testing last exit code
124 # vim: ts=4 sw=4 sts=4 et ft=sh :