2 #@IgnoreInspection BashAddShebang
3 # Activate robotframework virtualenv
4 # ${ROBOT_VENV} comes from the integration-install-robotframework.sh
6 # shellcheck source=${ROBOT_VENV}/bin/activate disable=SC1091
7 source "${ROBOT_VENV}/bin/activate"
8 source /tmp/common-functions.sh "${BUNDLEFOLDER}"
9 # Ensure we fail the job if any steps fail.
12 echo "#################################################"
13 echo "## Verify Cluster is UP ##"
14 echo "#################################################"
16 cat > "${WORKSPACE}/verify-cluster-is-up.sh" <<EOF
18 CONTROLLERID="member-\$1"
19 ODL_SYSTEM_IP_PATH=\$2
21 echo "Waiting for controller to come up..."
24 RESP="\$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null )"
26 SHARD="\$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/jolokia/read/org.opendaylight.controller:Category=Shards,name=\$CONTROLLERID-shard-inventory-config,type=DistributedConfigDatastore)"
28 if ([[ \$RESP == *"200"* ]] && [[ \$SHARD == *'"status":200'* ]]); then
31 elif (( "\$COUNT" > "600" )); then
32 echo Timeout Controller DOWN
33 echo "Dumping first 500K bytes of karaf log..."
34 head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
35 echo "Dumping last 500K bytes of karaf log..."
36 tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
37 echo "Listing all open ports on controller system"
41 COUNT=\$(( \${COUNT} + 1 ))
43 if [[ \$((\$COUNT % 5)) == 0 ]]; then
44 echo already waited \${COUNT} seconds...
49 echo "Listing all open ports on controller system.."
52 function exit_on_log_file_message {
53 echo "looking for \"\$1\" in log file"
54 if grep --quiet "\$1" "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"; then
55 echo ABORTING: found "\$1"
56 echo "Dumping first 500K bytes of karaf log..."
57 head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
58 echo "Dumping last 500K bytes of karaf log..."
59 tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
64 exit_on_log_file_message 'BindException: Address already in use'
65 exit_on_log_file_message 'server is unhealthy'
69 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
71 CONTROLLERIP=ODL_SYSTEM_${i}_IP
72 echo "Verifying member-${i} with IP address ${!CONTROLLERIP} is UP"
73 scp "${WORKSPACE}/verify-cluster-is-up.sh" "${!CONTROLLERIP}:/tmp"
74 # shellcheck disable=SC2029
75 ssh "${!CONTROLLERIP}" "bash /tmp/verify-cluster-is-up.sh ${i} ${!CONTROLLERIP}"
78 if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then
79 echo "Exiting without running tests to deploy openstack for testing"
83 if [ "${CONTROLLERSCOPE}" == 'all' ]; then
88 echo "Cool down for ${COOLDOWN_PERIOD} seconds :)..."
89 sleep "${COOLDOWN_PERIOD}"
91 echo "Generating controller variables..."
92 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
94 CONTROLLERIP=ODL_SYSTEM_${i}_IP
95 odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}"
96 echo "Lets's take the karaf thread dump"
97 ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_before.log"
98 pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_before.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
99 echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"
100 # shellcheck disable=SC2029
101 ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_before.log" || true
104 echo "Generating mininet variables..."
105 for i in $(seq 1 "${NUM_TOOLS_SYSTEM}")
107 MININETIP=TOOLS_SYSTEM_${i}_IP
108 tools_variables=${tools_variables}" -v ${MININETIP}:${!MININETIP}"
111 get_test_suites SUITES
113 echo "Starting Robot test suites ${SUITES} ..."
114 # ${TESTOPTIONS}, ${SUITES} are space-separated parameters and should not be quoted.
115 # shellcheck disable=SC2086
116 robot -N "${TESTPLAN}" \
117 --removekeywords wuks -c critical -e exclude -e "skip_if_${DISTROSTREAM}" \
118 -v BUNDLEFOLDER:"${BUNDLEFOLDER}" \
119 -v BUNDLE_URL:"${ACTUAL_BUNDLE_URL}" \
120 -v CONTROLLER:"${ODL_SYSTEM_IP}" \
121 -v CONTROLLER1:"${ODL_SYSTEM_2_IP}" \
122 -v CONTROLLER2:"${ODL_SYSTEM_3_IP}" \
123 -v CONTROLLER_USER:"${USER}" \
124 -v JAVA_HOME:"${JAVA_HOME}" \
125 -v JDKVERSION:"${JDKVERSION}" \
126 -v JENKINS_WORKSPACE:"${WORKSPACE}" \
127 -v MININET:"${TOOLS_SYSTEM_IP}" \
128 -v MININET1:"${TOOLS_SYSTEM_2_IP}" \
129 -v MININET2:"${TOOLS_SYSTEM_3_IP}" \
130 -v MININET_USER:"${USER}" \
131 -v NEXUSURL_PREFIX:"${NEXUSURL_PREFIX}" \
132 -v NUM_ODL_SYSTEM:"${NUM_ODL_SYSTEM}" \
133 -v NUM_TOOLS_SYSTEM:"${NUM_TOOLS_SYSTEM}" \
134 -v ODL_STREAM:"${DISTROSTREAM}" \
135 -v ODL_SYSTEM_IP:"${ODL_SYSTEM_IP}" ${odl_variables} \
136 -v ODL_SYSTEM_USER:"${USER}" \
137 -v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" ${tools_variables} \
138 -v TOOLS_SYSTEM_USER:"${USER}" \
139 -v USER_HOME:"${HOME}" \
141 ${TESTOPTIONS} ${SUITES} || true
145 echo "Examining the files in data/log and checking filesize"
146 # shellcheck disable=SC2029
147 ssh "${ODL_SYSTEM_1_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
148 # shellcheck disable=SC2029
149 ssh "${ODL_SYSTEM_1_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
150 # shellcheck disable=SC2029
151 ssh "${ODL_SYSTEM_2_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
152 # shellcheck disable=SC2029
153 ssh "${ODL_SYSTEM_2_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
154 # shellcheck disable=SC2029
155 ssh "${ODL_SYSTEM_3_IP}" "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
156 # shellcheck disable=SC2029
157 ssh "${ODL_SYSTEM_3_IP}" "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
159 set +e # We do not want to create red dot just because something went wrong while fetching logs.
160 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
162 CONTROLLERIP="ODL_SYSTEM_${i}_IP"
163 echo "Lets's take the karaf thread dump again"
164 ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_after.log"
165 pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_after.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
166 echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"
167 # shellcheck disable=SC2029
168 ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_after.log" || true
169 echo "killing karaf process..."
170 ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
173 for i in $(seq 1 "${NUM_ODL_SYSTEM}")
175 CONTROLLERIP=ODL_SYSTEM_${i}_IP
176 echo "Compressing karaf.log ${i}"
177 ssh "${!CONTROLLERIP}" gzip --best "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
178 echo "Fetching compressed karaf.log ${i}"
179 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.gz" "odl${i}_karaf.log.gz" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/karaf.log.gz"
180 # TODO: Should we compress the output log file as well?
181 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf_console.log" "odl${i}_karaf_console.log" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/karaf_console.log"
183 # FIXME: Put member index in filename, instead of directory name.
184 mkdir -p "gclogs-${i}"
185 scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/*.log" "gclogs-${i}/" && ssh "${!CONTROLLERIP}" rm -f "/tmp/${BUNDLEFOLDER}/data/log/*.log"
188 echo "Examine copied files"
191 true # perhaps Jenkins is testing last exit code
193 # vim: ts=4 sw=4 sts=4 et ft=sh :