2 #@IgnoreInspection BashAddShebang
3 # Activate robotframework virtualenv
4 # ${ROBOT_VENV} comes from the integration-install-robotframework.sh
6 # shellcheck source=${ROBOT_VENV}/bin/activate disable=SC1091
7 source "${ROBOT_VENV}/bin/activate"
8 source /tmp/common-functions.sh "${BUNDLEFOLDER}"
9 # Ensure we fail the job if any steps fail.
12 echo "#################################################"
13 echo "## Verify Cluster is UP ##"
14 echo "#################################################"
16 create_post_startup_script
18 copy_and_run_post_startup_script
20 dump_controller_threads
22 if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then
23 echo "Exiting without running tests to deploy openstack for testing"
27 echo "Generating controller variables..."
28 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
30 CONTROLLERIP=ODL_SYSTEM_${i}_IP
31 odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}"
34 echo "Generating mininet variables..."
35 for i in $(seq 1 "${NUM_TOOLS_SYSTEM}")
37 MININETIP=TOOLS_SYSTEM_${i}_IP
38 tools_variables=${tools_variables}" -v ${MININETIP}:${!MININETIP}"
41 get_test_suites SUITES
43 echo "Starting Robot test suites ${SUITES} ..."
44 # ${TESTOPTIONS}, ${SUITES} are space-separated parameters and should not be quoted.
45 # shellcheck disable=SC2086
46 robot -N "${TESTPLAN}" \
47 --removekeywords wuks -c critical -e exclude -e "skip_if_${DISTROSTREAM}" \
48 -v BUNDLEFOLDER:"${BUNDLEFOLDER}" \
49 -v BUNDLE_URL:"${ACTUAL_BUNDLE_URL}" \
50 -v CONTROLLER:"${ODL_SYSTEM_IP}" \
51 -v CONTROLLER1:"${ODL_SYSTEM_2_IP}" \
52 -v CONTROLLER2:"${ODL_SYSTEM_3_IP}" \
53 -v CONTROLLER_USER:"${USER}" \
54 -v JAVA_HOME:"${JAVA_HOME}" \
55 -v JDKVERSION:"${JDKVERSION}" \
56 -v JENKINS_WORKSPACE:"${WORKSPACE}" \
57 -v MININET:"${TOOLS_SYSTEM_IP}" \
58 -v MININET1:"${TOOLS_SYSTEM_2_IP}" \
59 -v MININET2:"${TOOLS_SYSTEM_3_IP}" \
60 -v MININET_USER:"${USER}" \
61 -v NEXUSURL_PREFIX:"${NEXUSURL_PREFIX}" \
62 -v NUM_ODL_SYSTEM:"${NUM_ODL_SYSTEM}" \
63 -v NUM_TOOLS_SYSTEM:"${NUM_TOOLS_SYSTEM}" \
64 -v ODL_STREAM:"${DISTROSTREAM}" \
65 -v ODL_SYSTEM_IP:"${ODL_SYSTEM_IP}" ${odl_variables} \
66 -v ODL_SYSTEM_USER:"${USER}" \
67 -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" ${tools_variables} \
68 -v TOOLS_SYSTEM_USER:"${USER}" \
69 -v USER_HOME:"${HOME}" \
71 ${TESTOPTIONS} ${SUITES} || true
75 echo "Examining the files in data/log and checking filesize"
76 # shellcheck disable=SC2029
77 ssh "${ODL_SYSTEM_1_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
78 # shellcheck disable=SC2029
79 ssh "${ODL_SYSTEM_1_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
80 # shellcheck disable=SC2029
81 ssh "${ODL_SYSTEM_2_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
82 # shellcheck disable=SC2029
83 ssh "${ODL_SYSTEM_2_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
84 # shellcheck disable=SC2029
85 ssh "${ODL_SYSTEM_3_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
86 # shellcheck disable=SC2029
87 ssh "${ODL_SYSTEM_3_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
89 set +e # We do not want to create red dot just because something went wrong while fetching logs.
90 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
92 CONTROLLERIP="ODL_SYSTEM_${i}_IP"
93 echo "Let's take the karaf thread dump again"
94 ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_after.log"
95 pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_after.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
96 echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"
97 # shellcheck disable=SC2029
98 ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_after.log" || true
99 echo "killing karaf process..."
100 ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
103 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
105 CONTROLLERIP=ODL_SYSTEM_${i}_IP
106 echo "Compressing karaf.log ${i}"
107 ssh "${!CONTROLLERIP}" gzip --best "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
108 echo "Fetching compressed karaf.log ${i}"
109 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.gz" "odl${i}_karaf.log.gz" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/karaf.log.gz"
110 # TODO: Should we compress the output log file as well?
111 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf_console.log" "odl${i}_karaf_console.log" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/karaf_console.log"
113 # FIXME: Put member index in filename, instead of directory name.
114 mkdir -p "gclogs-${i}"
115 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/*.log" "gclogs-${i}/" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/*.log"
118 echo "Examine copied files"
121 true # perhaps Jenkins is testing last exit code
123 # vim: ts=4 sw=4 sts=4 et ft=sh :