scp ${!CONTROLLERIP}:/tmp/odl${i}_zrpcd.log.tar ${NODE_FOLDER}
tar -xvf ${NODE_FOLDER}/odl${i}_karaf.log.tar -C ${NODE_FOLDER} --strip-components 2 --transform s/karaf/odl${i}_karaf/g
grep "ROBOT MESSAGE\| ERROR " ${NODE_FOLDER}/odl${i}_karaf.log > ${NODE_FOLDER}/odl${i}_err.log
- # Print ROBOT lines, Print Caused by...Exception: lines,
- # Print Exception{ lines as well as the previous line that has the timestamp for context
- sed -n -e '/ROBOT MESSAGE/P' -e '/Caused by.*Exception:/P' -e '$!N;/Exception:/P;D' -e '$!N;/Exception{/P;D' \
- ${NODE_FOLDER}/odl${i}_karaf.log > ${NODE_FOLDER}/odl${i}_exception.log
grep "ROBOT MESSAGE\| ERROR \| WARN \|Exception" \
${NODE_FOLDER}/odl${i}_karaf.log > ${NODE_FOLDER}/odl${i}_err_warn_exception.log
+ # Print ROBOT lines and print Exception lines. For exception lines also print the previous line for context
+ sed -n -e '/ROBOT MESSAGE/P' -e '$!N;/Exception/P;D' ${NODE_FOLDER}/odl${i}_karaf.log > ${NODE_FOLDER}/odl${i}_exception.log
rm ${NODE_FOLDER}/odl${i}_karaf.log.tar
mv karaf_${i}_threads* ${NODE_FOLDER}
mv ${NODE_FOLDER} ${WORKSPACE}/archives/
set -e
}
+# Check if rabbitmq is ready by looking for a pid in it's status.
+# The function returns the status of the grep command which callers can check.
+function is_rabbitmq_ready() {
+ local -r ip=$1
+ rm -f rabbit.txt
+ ${SSH} ${ip} "sudo rabbitmqctl status" > rabbit.txt
+ grep pid rabbit.txt
+}
+
+# retry the given command ($3) until success for a number of iterations ($1)
+# sleeping ($2) between tries.
+function retry() {
+ set +e
+ local -r -i max_tries=${1}
+ local -r -i sleep_time=${2}
+ local -r cmd=${3}
+ local -i retries=1
+ local -i rc=1
+ while true; do
+ echo "retry ${cmd}: attempt: ${retries}"
+ ${cmd}
+ rc=$?
+ if ((${rc} == 0)); then
+ break;
+ else
+ if ((${retries} == ${max_tries})); then
+ break
+ else
+ ((retries++))
+ sleep ${sleep_time}
+ fi
+ fi
+ done
+ set -e
+ return ${rc}
+}
+
# if we are using the new netvirt impl, as determined by the feature name
# odl-netvirt-openstack (note: old impl is odl-ovsdb-openstack) then we
# want PROVIDER_MAPPINGS to be used -- this should be fixed if we want to support
# AccessRefused: (0, 0): (403) ACCESS_REFUSED - Login was refused using authentication mechanism AMQPLAIN. For details see the broker logfile.
# Compare that timestamp to this log in the control stack.log: sudo rabbitmqctl set_permissions -p nova_cell1 stackrabbit
# If the n-cpu.log is earlier than the control stack.log timestamp then the failure condition is likely hit.
-# TODO: modify devstack to wait for rabbitmq to be available on the controller before starting nova-compute.
-echo "Sleeping for 360s to allow controller to create nova_cell1 before the computes need it"
-sleep 360
+
+echo "Wait a maximum of 30m until rabbitmq is ready to allow the controller to create nova_cell1 before the computes need it"
+retry 30 60 "is_rabbitmq_ready ${OPENSTACK_CONTROL_NODE_1_IP}"
+rc=$?
+if ((${rc} == 0)); then
+ echo "rabbitmq is ready, starting ${NUM_OPENSTACK_COMPUTE_NODES} compute(s)"
+else
+ echo "rabbitmq was not ready in "
+ collect_logs
+ exit 1
+fi
for i in `seq 1 ${NUM_OPENSTACK_COMPUTE_NODES}`; do
NUM_COMPUTES_PER_SITE=$((NUM_OPENSTACK_COMPUTE_NODES / NUM_OPENSTACK_SITES))
sudo ip netns exec pnf_ns ifconfig pnf_veth1 up ${EXTNET_PNF_IP}/24;
sudo ovs-vsctl add-port ${PUBLIC_BRIDGE} pnf_veth0;
"
+ # Control Node - set VXLAN TEP IP for Genius Auto TZ
+ ${SSH} ${!CONTROLIP} "
+ sudo ovs-vsctl set O . external_ids:tep-ip=${!CONTROLIP};
+ "
# Control Node - external net internet address simulation
${SSH} ${!CONTROLIP} "
${SSH} $compute_ip "
sudo ovs-vsctl add-port $PUBLIC_BRIDGE $CONTROLPORT -- set interface $CONTROLPORT type=vxlan options:local_ip=$compute_ip options:remote_ip=${!CONTROLIP} options:dst_port=9876 options:key=flow
"
+ #Compute Node - set VXLAN TEP IP for Genius Auto TZ
+ ${SSH} $compute_ip "
+ sudo ovs-vsctl set O . external_ids:tep-ip=${compute_ip};
+ "
done
done
# Use the testplan if specific SUITES are not defined.
if [ -z "${SUITES}" ]; then
SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
+else
+ newsuites=""
+ workpath="${WORKSPACE}/test/csit/suites"
+ for suite in ${SUITES}; do
+ fullsuite="${workpath}/${suite}"
+ if [ -z ${newsuites} ]; then
+ newsuites+=${fullsuite}
+ else
+ newsuites+=" "${fullsuite}
+ fi
+ done
+ SUITES=${newsuites}
fi
# TODO: run openrc on control node and then scrape the vars from it
echo "Starting Robot test suites ${SUITES} ..."
# please add pybot -v arguments on a single line and alphabetized
-pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude -e skip_if_${DISTROSTREAM} \
+for suite in ${SUITES}; do
+ log_name_ext=$(basename ${suite} | cut -d. -f1)
+ pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude -e skip_if_${DISTROSTREAM} \
+ --log log_${log_name_ext}.html --report None --output output_${log_name_ext}.xml \
-v BUNDLEFOLDER:${BUNDLEFOLDER} \
-v BUNDLE_URL:${ACTUAL_BUNDLE_URL} \
-v CONTROLLER_USER:${USER} \
-v TOOLS_SYSTEM_2_IP:${TOOLS_SYSTEM_2_IP} \
-v USER_HOME:${HOME} \
-v WORKSPACE:/tmp \
- ${TESTOPTIONS} ${SUITES} || true
+ ${TESTOPTIONS} ${suite} || true
+done
+#rebot exit codes seem to be different
+rebot --output ${WORKSPACE}/output.xml --log None --report None output_*.xml || true
echo "Examining the files in data/log and checking file size"
ssh ${ODL_SYSTEM_IP} "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
Code Review / releng / builder.git / commitdiff