local prog="$0"
local lastline="$1"
local lasterr="$2"
- echo "${prog}: line ${lastline}: exit status of last command: ${lasterr}"
- echo "command: ${BASH_COMMAND}"
+ echo "trap_hanlder: ${prog}: line ${lastline}: exit status of last command: ${lasterr}"
+ echo "trap_handler: command: ${BASH_COMMAND}"
collect_logs
exit 1
} # trap_handler()
if [ "${ENABLE_NETWORKING_L2GW}" == "yes" ]; then
#networking-l2gw is not officially available in any release yet. Gettting the latest stable version.
- $PYTHON -m pip install networking-l2gw
+ $PYTHON -m pip install networking-l2gw==11.0.0
fi
}
} # configure_haproxy_for_neutron_requests()
# Collect the list of files on the hosts
-function list_files () {
- local ip=$1
- local folder=$2
- ${SSH} ${ip} "sudo find /etc > /tmp/find.etc.txt"
- ${SSH} ${ip} "sudo find /opt/stack > /tmp/find.opt.stack.txt"
- ${SSH} ${ip} "sudo find /var > /tmp/find2.txt"
- ${SSH} ${ip} "sudo find /var > /tmp/find.var.txt"
- rsync --rsync-path="sudo rsync" --list-only -arvhe ssh ${ip}:/etc/ > ${folder}/rsync.etc.txt
- rsync --rsync-path="sudo rsync" --list-only -arvhe ssh ${ip}:/opt/stack/ > ${folder}/rsync.opt.stack.txt
- rsync --rsync-path="sudo rsync" --list-only -arvhe ssh ${ip}:/var/ > ${folder}/rsync.var.txt
- scp ${ip}:/tmp/find.etc.txt ${folder}
- scp ${ip}:/tmp/find.opt.stack.txt ${folder}
- scp ${ip}:/tmp/find2.txt ${folder}
- scp ${ip}:/tmp/find.var.txt ${folder}
+function collect_files () {
+ local -r ip=$1
+ local -r folder=$2
+ finddir=/tmp/finder
+ ${SSH} ${ip} "mkdir -p ${finddir}"
+ ${SSH} ${ip} "sudo find /etc > ${finddir}/find.etc.txt"
+ ${SSH} ${ip} "sudo find /opt/stack > ${finddir}/find.opt.stack.txt"
+ ${SSH} ${ip} "sudo find /var > ${finddir}/find2.txt"
+ ${SSH} ${ip} "sudo find /var > ${finddir}/find.var.txt"
+ ${SSH} ${ip} "sudo tar -cf - -C /tmp finder | xz -T 0 > /tmp/find.tar.xz"
+ scp ${ip}:/tmp/find.tar.xz ${folder}
+ mkdir -p ${finddir}
+ rsync --rsync-path="sudo rsync" --list-only -arvhe ssh ${ip}:/etc/ > ${finddir}/rsync.etc.txt
+ rsync --rsync-path="sudo rsync" --list-only -arvhe ssh ${ip}:/opt/stack/ > ${finddir}/rsync.opt.stack.txt
+ rsync --rsync-path="sudo rsync" --list-only -arvhe ssh ${ip}:/var/ > ${finddir}/rsync.var.txt
+ tar -cf - -C /tmp finder | xz -T 0 > /tmp/rsync.tar.xz
+ cp /tmp/rsync.tar.xz ${folder}
}
function collect_logs () {
scp ${!CONTROLLERIP}:/tmp/journalctl.log ${NODE_FOLDER}
${SSH} ${!CONTROLLERIP} "dmesg -T > /tmp/dmesg.log"
scp ${!CONTROLLERIP}:/tmp/dmesg.log ${NODE_FOLDER}
- rsync -avhe ssh ${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/etc ${NODE_FOLDER}
+ ${SSH} ${!CONTROLLERIP} "tar -cf - -C /tmp/${BUNDLEFOLDER} etc | xz -T 0 > /tmp/etc.tar.xz"
+ scp ${!CONTROLLERIP}:/tmp/etc.tar.xz ${NODE_FOLDER}
${SSH} ${!CONTROLLERIP} "cp -r /tmp/${BUNDLEFOLDER}/data/log /tmp/odl_log"
${SSH} ${!CONTROLLERIP} "tar -cf /tmp/odl${i}_karaf.log.tar /tmp/odl_log/*"
scp ${!CONTROLLERIP}:/tmp/odl${i}_karaf.log.tar ${NODE_FOLDER}
scp ${!OSIP}:/tmp/ovsdb-tool.log ${NODE_FOLDER}
scp ${!OSIP}:/var/log/openvswitch/ovs-vswitchd.log ${NODE_FOLDER}
scp ${!OSIP}:/var/log/openvswitch/ovsdb-server.log ${NODE_FOLDER}
- list_files "${!OSIP}" "${NODE_FOLDER}"
+ collect_files "${!OSIP}" "${NODE_FOLDER}"
+ ${SSH} ${!OSIP} "sudo tar -cf - -C /var/log rabbitmq | xz -T 0 > /tmp/rabbitmq.tar.xz "
+ scp ${!OSIP}:/tmp/rabbitmq.tar.xz ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/etc/hosts ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/usr/lib/systemd/system/haproxy.service ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/audit/audit.log ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/httpd/keystone_access.log ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/httpd/keystone.log ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/messages ${NODE_FOLDER}
- rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/rabbitmq ${NODE_FOLDER}
rsync -avhe ssh ${!OSIP}:/opt/stack/logs/* ${NODE_FOLDER} # rsync to prevent copying of symbolic links
mv local.conf_control_${!OSIP} ${NODE_FOLDER}/local.conf
- # qdhcp files are created by robot tests
- mv /tmp/qdhcp ${NODE_FOLDER}
+ # qdhcp files are created by robot tests and copied into /tmp/qdhcp during the test
+ tar -cf - -C /tmp qdhcp | xz -T 0 > /tmp/qdhcp.tar.xz
+ mv /tmp/qdhcp.tar.xz ${NODE_FOLDER}
mv ${NODE_FOLDER} ${WORKSPACE}/archives/
done
scp ${!OSIP}:/tmp/ovsdb-tool.log ${NODE_FOLDER}
scp ${!OSIP}:/var/log/openvswitch/ovs-vswitchd.log ${NODE_FOLDER}
scp ${!OSIP}:/var/log/openvswitch/ovsdb-server.log ${NODE_FOLDER}
- list_files "${!OSIP}" "${NODE_FOLDER}"
+ collect_files "${!OSIP}" "${NODE_FOLDER}"
+ ${SSH} ${!OSIP} "sudo tar -cf - -C /var/log libvirt | xz -T 0 > /tmp/libvirt.tar.xz "
+ scp ${!OSIP}:/tmp/libvirt.tar.xz ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/etc/hosts ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/audit/audit.log ${NODE_FOLDER}
- rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/libvirt ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/messages ${NODE_FOLDER}
rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/nova-agent.log ${NODE_FOLDER}
rsync -avhe ssh ${!OSIP}:/opt/stack/logs/* ${NODE_FOLDER} # rsync to prevent copying of symbolic links
# retry the given command ($3) until success for a number of iterations ($1)
# sleeping ($2) between tries.
function retry() {
- set +e
local -r -i max_tries=${1}
local -r -i sleep_time=${2}
local -r cmd=${3}
fi
fi
done
- set -e
return ${rc}
}
# Additional services
CORE_OS_CONTROL_SERVICES+=",mysql,rabbit"
+# collect performance stats
+CORE_OS_COMPUTE_SERVICES="dstat"
# computes only need nova and odl
-CORE_OS_COMPUTE_SERVICES="n-cpu,odl-compute"
+CORE_OS_COMPUTE_SERVICES+=",n-cpu,odl-compute"
cat > ${WORKSPACE}/disable_firewall.sh << EOF
sudo systemctl stop firewalld
# Compare that timestamp to this log in the control stack.log: sudo rabbitmqctl set_permissions -p nova_cell1 stackrabbit
# If the n-cpu.log is earlier than the control stack.log timestamp then the failure condition is likely hit.
if [ ${NUM_OPENSTACK_COMPUTE_NODES} -gt 0 ]; then
- WAIT_FOR_RABBITMQ_MINUTES=60
- echo "Wait a maximum of ${WAIT_FOR_RABBITMQ_MINUTES}m until rabbitmq is ready to allow the controller to create nova_cell1 before the computes need it"
- retry ${WAIT_FOR_RABBITMQ_MINUTES} 60 "is_rabbitmq_ready ${OPENSTACK_CONTROL_NODE_1_IP}"
- rc=$?
- if ((${rc} == 0)); then
+ WAIT_FOR_RABBITMQ_MINUTES=60
+ echo "Wait a maximum of ${WAIT_FOR_RABBITMQ_MINUTES}m until rabbitmq is ready to allow the controller to create nova_cell1 before the computes need it"
+ set +e
+ retry ${WAIT_FOR_RABBITMQ_MINUTES} 60 "is_rabbitmq_ready ${OPENSTACK_CONTROL_NODE_1_IP}"
+ rc=$?
+ set -e
+ if ((${rc} == 0)); then
echo "rabbitmq is ready, starting ${NUM_OPENSTACK_COMPUTE_NODES} compute(s)"
- else
+ else
echo "rabbitmq was not ready in ${WAIT_FOR_RABBITMQ_MINUTES}m"
collect_logs
exit 1
- fi
+ fi
fi
for i in `seq 1 ${NUM_OPENSTACK_COMPUTE_NODES}`; do
sudo ip netns exec pnf_ns ifconfig pnf_veth1 up ${EXTNET_PNF_IP}/24;
sudo ovs-vsctl add-port ${PUBLIC_BRIDGE} pnf_veth0;
"
- # Control Node - set VXLAN TEP IP for Genius Auto TZ
- ${SSH} ${!CONTROLIP} "
- sudo ovs-vsctl set O . external_ids:tep-ip=${!CONTROLIP};
- "
# Control Node - external net internet address simulation
${SSH} ${!CONTROLIP} "
# please add pybot -v arguments on a single line and alphabetized
suite_num=0
for suite in ${SUITES}; do
- # prepend a incrmental counter to the suite name so that the full robot log combining all the suites as is done
- # in the rebot step below will list all the suites in chronological order as rebot seems to alphabatize them
+ # prepend an incremental counter to the suite name so that the full robot log combining all the suites as is done
+ # in the rebot step below will list all the suites in chronological order as rebot seems to alphabetize them
let "suite_num = suite_num + 1"
suite_index="$(printf %02d ${suite_num})"
suite_name="$(basename ${suite} | cut -d. -f1)"
log_name="${suite_index}_${suite_name}"
- pybot -N ${log_name} --removekeywords wuks -c critical -e exclude -e skip_if_${DISTROSTREAM} \
+ pybot -N ${log_name} \
+ -c critical -e exclude -e skip_if_${DISTROSTREAM} \
--log log_${log_name}.html --report None --output output_${log_name}.xml \
+ --removekeywords wuks \
+ --removekeywords name:SetupUtils.Setup_Utils_For_Setup_And_Teardown \
+ --removekeywords name:SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing \
+ --removekeywords name:OpenStackOperations.Add_OVS_Logging_On_All_OpenStack_Nodes \
-v BUNDLEFOLDER:${BUNDLEFOLDER} \
-v BUNDLE_URL:${ACTUAL_BUNDLE_URL} \
-v CONTROLLERFEATURES:"${CONTROLLERFEATURES}" \
Code Review / releng / builder.git / blobdiff