Merge "Update Ubuntu 14.40 image for GBP"

[releng/builder.git] / jjb / integration / common-functions.sh

diff --git a/jjb/integration/common-functions.sh b/jjb/integration/common-functions.sh
index bdec49d53137ef566cd165f89250a4488a9ba3a8..e2066b4153ce550c89515ef9fe8eec664e27d983 100644 (file)
--- a/jjb/integration/common-functions.sh
+++ b/jjb/integration/common-functions.sh
@@ -117,6 +117,22 @@ function configure_karaf_log() {
     cat ${LOGCONF}
 } # function configure_karaf_log()
 
+function get_os_deploy() {
+    local -r num_systems=${1:-$NUM_OPENSTACK_SYSTEM}
+    case ${num_systems} in
+    1)
+        OS_DEPLOY="1cmb-0ctl-0cmp"
+        ;;
+    2)
+        OS_DEPLOY="1cmb-0ctl-1cmp"
+        ;;
+    3|*)
+        OS_DEPLOY="0cmb-1ctl-2cmp"
+        ;;
+    esac
+    export OS_DEPLOY
+}
+
 function run_plan() {
     local -r type=$1
 
@@ -152,6 +168,31 @@ function run_plan() {
     printf "Finished running ${type} plans\n"
 } # function run_plan()
 
+# Return elapsed time. Usage:
+# - Call first time with no arguments and a new timer is returned.
+# - Next call with the first argument as the timer and the elapsed time is returned.
+function timer()
+{
+    if [ $# -eq 0 ]; then
+        # return the current time
+        printf "$(date "+%s")"
+    else
+        local start_time=$1
+        end_time=$(date "+%s")
+
+        if [ -z "$start_time" ]; then
+            start_time=$end_time;
+        fi
+
+        delta_time=$((end_time - start_time))
+        ds=$((delta_time % 60))
+        dm=$(((delta_time / 60) % 60))
+        dh=$((delta_time / 3600))
+        # return the elapsed time
+        printf "%d:%02d:%02d" $dh $dm $ds
+    fi
+}
+
 # convert commas in csv strings to spaces (ssv)
 function csv2ssv() {
     local csv=$1
@@ -162,6 +203,17 @@ function csv2ssv() {
     echo "${ssv}"
 } # csv2ssv
 
+function is_openstack_feature_enabled() {
+    local feature=$1
+    for enabled_feature in $(csv2ssv ${ENABLE_OS_SERVICES}); do
+        if [ "${enabled_feature}" == "${feature}" ]; then
+           echo 1
+           return
+        fi
+    done
+    echo 0
+}
+
 SSH="ssh -t -t"
 
 # shellcheck disable=SC2153
@@ -189,7 +241,6 @@ ODL_ML2_PORT_BINDING: ${ODL_ML2_PORT_BINDING}
 DEVSTACK_KUBERNETES_PLUGIN_REPO: ${DEVSTACK_KUBERNETES_PLUGIN_REPO}
 DEVSTACK_LBAAS_PLUGIN_REPO: ${DEVSTACK_LBAAS_PLUGIN_REPO}
 DEVSTACK_NETWORKING_SFC_PLUGIN_REPO: ${DEVSTACK_NETWORKING_SFC_PLUGIN_REPO}
-ODL_ENABLE_L3_FWD: ${ODL_ENABLE_L3_FWD}
 IPSEC_VXLAN_TUNNELS_ENABLED: ${IPSEC_VXLAN_TUNNELS_ENABLED}
 PUBLIC_BRIDGE: ${PUBLIC_BRIDGE}
 ENABLE_HAPROXY_FOR_NEUTRON: ${ENABLE_HAPROXY_FOR_NEUTRON}
@@ -212,6 +263,28 @@ ODL_SNAT_MODE: ${ODL_SNAT_MODE}
 EOF
 }
 
+function tcpdump_start() {
+    local -r prefix=$1
+    local -r ip=$2
+    local -r filter=$3
+    filter_=${filter// /_}
+
+    printf "node ${ip}, ${prefix}_${ip}__${filter}: starting tcpdump\n"
+    ssh ${ip} "nohup sudo /usr/sbin/tcpdump -vvv -ni eth0 ${filter} -w /tmp/tcpdump_${prefix}_${ip}__${filter_}.pcap > /tmp/tcpdump_start.log 2>&1 &"
+    ${SSH} ${ip} "ps -ef | grep tcpdump"
+}
+
+function tcpdump_stop() {
+    local -r ip=$1
+
+    printf "node $ip: stopping tcpdump\n"
+    ${SSH} ${ip} "ps -ef | grep tcpdump.sh"
+    ${SSH} ${ip} "sudo pkill -f tcpdump"
+    ${SSH} ${ip} "sudo xz -9ekvvf /tmp/*.pcap"
+    ${SSH} ${ip} "sudo ls -al /tmp/*.pcap"
+    # copy_logs will copy any *.xz files
+}
+
 # Collect the list of files on the hosts
 function collect_files() {
     local -r ip=$1
@@ -391,10 +464,23 @@ EOF
     # Control Node
     for i in `seq 1 ${NUM_OPENSTACK_CONTROL_NODES}`; do
         OSIP=OPENSTACK_CONTROL_NODE_${i}_IP
-        echo "collect_logs: for openstack control node ip: ${!OSIP}"
-        NODE_FOLDER="control_${i}"
+        if [ "$(is_openstack_feature_enabled n-cpu)" == "1" ]; then
+            echo "collect_logs: for openstack combo node ip: ${!OSIP}"
+            NODE_FOLDER="combo_${i}"
+        else
+            echo "collect_logs: for openstack control node ip: ${!OSIP}"
+            NODE_FOLDER="control_${i}"
+        fi
         mkdir -p ${NODE_FOLDER}
+        tcpdump_stop "${!OSIP}"
         scp extra_debug.sh ${!OSIP}:/tmp
+        # Capture compute logs if this is a combo node
+        if [ "$(is_openstack_feature_enabled n-cpu)" == "1" ]; then
+            scp ${!OSIP}:/etc/nova/nova.conf ${NODE_FOLDER}
+            scp ${!OSIP}:/etc/nova/nova-cpu.conf ${NODE_FOLDER}
+            scp ${!OSIP}:/etc/openstack/clouds.yaml ${NODE_FOLDER}
+            rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/nova-agent.log ${NODE_FOLDER}
+        fi
         ${SSH} ${!OSIP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log 2>&1"
         scp ${!OSIP}:/etc/dnsmasq.conf ${NODE_FOLDER}
         scp ${!OSIP}:/etc/keystone/keystone.conf ${NODE_FOLDER}
@@ -424,6 +510,7 @@ EOF
         scp ${!OSIP}:/tmp/get_devstack.sh.txt ${NODE_FOLDER}
         scp ${!OSIP}:/tmp/journalctl.log ${NODE_FOLDER}
         scp ${!OSIP}:/tmp/ovsdb-tool.log ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/tcpdump_start.log ${NODE_FOLDER}
         collect_files "${!OSIP}" "${NODE_FOLDER}"
         ${SSH} ${!OSIP} "sudo tar -cf - -C /var/log rabbitmq | xz -T 0 > /tmp/rabbitmq.tar.xz "
         scp ${!OSIP}:/tmp/rabbitmq.tar.xz ${NODE_FOLDER}
@@ -449,6 +536,7 @@ EOF
         echo "collect_logs: for openstack compute node ip: ${!OSIP}"
         NODE_FOLDER="compute_${i}"
         mkdir -p ${NODE_FOLDER}
+        tcpdump_stop "${!OSIP}"
         scp extra_debug.sh ${!OSIP}:/tmp
         ${SSH} ${!OSIP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log 2>&1"
         scp ${!OSIP}:/etc/nova/nova.conf ${NODE_FOLDER}
@@ -464,6 +552,7 @@ EOF
         scp ${!OSIP}:/tmp/get_devstack.sh.txt ${NODE_FOLDER}
         scp ${!OSIP}:/tmp/journalctl.log ${NODE_FOLDER}
         scp ${!OSIP}:/tmp/ovsdb-tool.log ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/tcpdump_start.log ${NODE_FOLDER}
         collect_files "${!OSIP}" "${NODE_FOLDER}"
         ${SSH} ${!OSIP} "sudo tar -cf - -C /var/log libvirt | xz -T 0 > /tmp/libvirt.tar.xz "
         scp ${!OSIP}:/tmp/libvirt.tar.xz ${NODE_FOLDER}