# ${ROBOT_VENV} comes from the integration-install-robotframework.sh
# script.
# shellcheck source=${ROBOT_VENV}/bin/activate disable=SC1091
-source "${ROBOT_VENV}/bin/activate"
+. "${ROBOT_VENV}/bin/activate"
source /tmp/common-functions.sh "${BUNDLEFOLDER}"
# Ensure we fail the job if any steps fail.
set -ex -o pipefail
echo "## Verify Cluster is UP ##"
echo "#################################################"
-cat > "${WORKSPACE}/verify-cluster-is-up.sh" <<EOF
-
-CONTROLLERID="member-\$1"
-ODL_SYSTEM_IP_PATH=\$2
-
-echo "Waiting for controller to come up..."
-COUNT="0"
-while true; do
- RESP="\$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null )"
- echo \$RESP
- SHARD="\$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/jolokia/read/org.opendaylight.controller:Category=Shards,name=\$CONTROLLERID-shard-inventory-config,type=DistributedConfigDatastore)"
- echo \$SHARD
- if ([[ \$RESP == *"200"* ]] && [[ \$SHARD == *'"status":200'* ]]); then
- echo Controller is UP
- break
- elif (( "\$COUNT" > "600" )); then
- echo Timeout Controller DOWN
- echo "Dumping first 500K bytes of karaf log..."
- head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
- echo "Dumping last 500K bytes of karaf log..."
- tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
- echo "Listing all open ports on controller system"
- netstat -pnatu
- exit 1
- else
- COUNT=\$(( \${COUNT} + 1 ))
- sleep 1
- if [[ \$((\$COUNT % 5)) == 0 ]]; then
- echo already waited \${COUNT} seconds...
- fi
- fi
-done
-
-echo "Listing all open ports on controller system.."
-netstat -pnatu
+create_post_startup_script
-function exit_on_log_file_message {
- echo "looking for \"\$1\" in log file"
- if grep --quiet "\$1" "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"; then
- echo ABORTING: found "\$1"
- echo "Dumping first 500K bytes of karaf log..."
- head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
- echo "Dumping last 500K bytes of karaf log..."
- tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
- exit 1
- fi
-}
+copy_and_run_post_startup_script
-exit_on_log_file_message 'BindException: Address already in use'
-exit_on_log_file_message 'server is unhealthy'
-
-EOF
-
-for i in $(seq 1 "${NUM_ODL_SYSTEM}")
-do
- CONTROLLERIP=ODL_SYSTEM_${i}_IP
- echo "Verifying member-${i} with IP address ${!CONTROLLERIP} is UP"
- scp "${WORKSPACE}/verify-cluster-is-up.sh" "${!CONTROLLERIP}:/tmp"
- # shellcheck disable=SC2029
- ssh "${!CONTROLLERIP}" "bash /tmp/verify-cluster-is-up.sh ${i} ${!CONTROLLERIP}"
-done
+dump_controller_threads
if [ "${NUM_OPENSTACK_SYSTEM}" -gt 0 ]; then
echo "Exiting without running tests to deploy openstack for testing"
exit
fi
-if [ "${CONTROLLERSCOPE}" == 'all' ]; then
- COOLDOWN_PERIOD="180"
-else
- COOLDOWN_PERIOD="60"
-fi
-echo "Cool down for ${COOLDOWN_PERIOD} seconds :)..."
-sleep "${COOLDOWN_PERIOD}"
-
echo "Generating controller variables..."
for i in $(seq 1 "${NUM_ODL_SYSTEM}")
do
CONTROLLERIP=ODL_SYSTEM_${i}_IP
odl_variables=${odl_variables}" -v ${CONTROLLERIP}:${!CONTROLLERIP}"
- echo "Lets's take the karaf thread dump"
- ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_before.log"
- pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_before.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
- echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"
- # shellcheck disable=SC2029
- ssh "${!CONTROLLERIP}" "${JAVA_HOME}/bin/jstack -l ${pid}" > "${WORKSPACE}/karaf_${i}_${pid}_threads_before.log" || true
done
echo "Generating mininet variables..."
# ${TESTOPTIONS}, ${SUITES} are space-separated parameters and should not be quoted.
# shellcheck disable=SC2086
robot -N "${TESTPLAN}" \
- --removekeywords wuks -c critical -e exclude -e "skip_if_${DISTROSTREAM}" \
+ --removekeywords wuks -e exclude -e "skip_if_${DISTROSTREAM}" \
-v BUNDLEFOLDER:"${BUNDLEFOLDER}" \
-v BUNDLE_URL:"${ACTUAL_BUNDLE_URL}" \
-v CONTROLLER:"${ODL_SYSTEM_IP}" \
-v TOOLS_SYSTEM_IP:"${TOOLS_SYSTEM_IP}" ${tools_variables} \
-v TOOLS_SYSTEM_USER:"${USER}" \
-v USER_HOME:"${HOME}" \
+ -v IS_KARAF_APPL:"${IS_KARAF_APPL}" \
-v WORKSPACE:/tmp \
${TESTOPTIONS} ${SUITES} || true
for i in $(seq 1 "${NUM_ODL_SYSTEM}")
do
CONTROLLERIP="ODL_SYSTEM_${i}_IP"
- echo "Lets's take the karaf thread dump again"
+ echo "Let's take the karaf thread dump again"
ssh "${!CONTROLLERIP}" "sudo ps aux" > "${WORKSPACE}/ps_after.log"
pid=$(grep org.apache.karaf.main.Main "${WORKSPACE}/ps_after.log" | grep -v grep | tr -s ' ' | cut -f2 -d' ')
echo "karaf main: org.apache.karaf.main.Main, pid:${pid}"