Add function to collect lists of files
[releng/builder.git] / jjb / integration / integration-deploy-openstack-run-test.sh
index a8f3a817de1482e4a6b53ca30a332a304f444f59..f18f683e2097166bb7130bc0799d539640d062c0 100644 (file)
@@ -14,10 +14,11 @@ $PYTHON -m pip install --upgrade urllib3
 python ${WORKSPACE}/test/tools/distchanges/changes.py -d /tmp/distribution_folder \
                   -u ${ACTUAL_BUNDLE_URL} -b ${DISTROBRANCH} \
                   -r ssh://jenkins-${SILO}@git.opendaylight.org:29418 || true
-
-echo "#################################################"
-echo "##         Deploy Openstack 3-node             ##"
-echo "#################################################"
+cat << EOF
+#################################################
+##         Deploy Openstack 3-node             ##
+#################################################
+EOF
 
 # Catch command errors and collect logs.
 # This ensures logs are collected when script commands fail rather than simply exiting.
@@ -33,6 +34,50 @@ function trap_handler() {
 
 trap 'trap_handler ${LINENO} ${$?}' ERR
 
+function print_job_parameters() {
+    cat << EOF
+
+Job parameters:
+DISTROBRANCH: ${DISTROBRANCH}
+DISTROSTREAM: ${DISTROSTREAM}
+BUNDLE_URL: ${BUNDLE_URL}
+CONTROLLERFEATURES: ${CONTROLLERFEATURES}
+CONTROLLERDEBUGMAP: ${CONTROLLERDEBUGMAP}
+TESTPLAN: ${TESTPLAN}
+SUITES: ${SUITES}
+PATCHREFSPEC: ${PATCHREFSPEC}
+OPENSTACK_BRANCH: ${OPENSTACK_BRANCH}
+DEVSTACK_HASH: ${DEVSTACK_HASH}
+ODL_ML2_DRIVER_REPO: ${ODL_ML2_DRIVER_REPO}
+ODL_ML2_BRANCH: ${ODL_ML2_BRANCH}
+ODL_ML2_DRIVER_VERSION: ${ODL_ML2_DRIVER_VERSION}
+ODL_ML2_PORT_BINDING: ${ODL_ML2_PORT_BINDING}
+DEVSTACK_KUBERNETES_PLUGIN_REPO: ${DEVSTACK_KUBERNETES_PLUGIN_REPO}
+DEVSTACK_LBAAS_PLUGIN_REPO: ${DEVSTACK_LBAAS_PLUGIN_REPO}
+DEVSTACK_NETWORKING_SFC_PLUGIN_REPO: ${DEVSTACK_NETWORKING_SFC_PLUGIN_REPO}
+ODL_ENABLE_L3_FWD: ${ODL_ENABLE_L3_FWD}
+IPSEC_VXLAN_TUNNELS_ENABLED: ${IPSEC_VXLAN_TUNNELS_ENABLED}
+PUBLIC_BRIDGE: ${PUBLIC_BRIDGE}
+ENABLE_HAPROXY_FOR_NEUTRON: ${ENABLE_HAPROXY_FOR_NEUTRON}
+ENABLE_OS_SERVICES: ${ENABLE_OS_SERVICES}
+ENABLE_OS_COMPUTE_SERVICES: ${ENABLE_OS_COMPUTE_SERVICES}
+ENABLE_OS_PLUGINS: ${ENABLE_OS_PLUGINS}
+DISABLE_OS_SERVICES: ${DISABLE_OS_SERVICES}
+TENANT_NETWORK_TYPE: ${TENANT_NETWORK_TYPE}
+SECURITY_GROUP_MODE: ${SECURITY_GROUP_MODE}
+PUBLIC_PHYSICAL_NETWORK: ${PUBLIC_PHYSICAL_NETWORK}
+ENABLE_NETWORKING_L2GW: ${ENABLE_NETWORKING_L2GW}
+CREATE_INITIAL_NETWORKS: ${CREATE_INITIAL_NETWORKS}
+LBAAS_SERVICE_PROVIDER: ${LBAAS_SERVICE_PROVIDER}
+NUM_OPENSTACK_SITES: ${NUM_OPENSTACK_SITES}
+ODL_SFC_DRIVER: ${ODL_SFC_DRIVER}
+ODL_SNAT_MODE: ${ODL_SNAT_MODE}
+
+EOF
+}
+
+print_job_parameters
+
 function create_etc_hosts() {
     NODE_IP=$1
     CTRL_IP=$2
@@ -63,6 +108,28 @@ function csv2ssv() {
     echo "${ssv}"
 } # csv2ssv
 
+# Add enable_services and disable_services to the local.conf
+function add_os_services() {
+    local core_services=$1
+    local enable_services=$2
+    local disable_services=$3
+    local local_conf_file_name=$4
+
+    cat >> ${local_conf_file_name} << EOF
+enable_service $(csv2ssv "${core_services}")
+EOF
+    if [ -n "${enable_services}" ]; then
+        cat >> ${local_conf_file_name} << EOF
+enable_service $(csv2ssv "${enable_services}")
+EOF
+    fi
+    if [ -n "${disable_services}" ]; then
+        cat >> ${local_conf_file_name} << EOF
+disable_service $(csv2ssv "${disable_services}")
+EOF
+    fi
+}
+
 function create_control_node_local_conf() {
     HOSTIP=$1
     MGRIP=$2
@@ -80,20 +147,7 @@ RECLONE=${RECLONE}
 disable_all_services
 EOF
 
-    CORE_OS_SERVICES="dstat,g-api,g-reg,key,mysql,n-api,n-cauth,n-cond,n-crt,n-obj,n-sch,odl-compute,odl-neutron,q-dhcp,q-meta,q-svc,rabbit"
-    cat >> ${local_conf_file_name} << EOF
-enable_service $(csv2ssv "${CORE_OS_SERVICES}")
-EOF
-    if [ -n "${ENABLE_OS_SERVICES}" ]; then
-        cat >> ${local_conf_file_name} << EOF
-enable_service $(csv2ssv "${ENABLE_OS_SERVICES}")
-EOF
-    fi
-    if [ -n "${DISABLE_OS_SERVICES}" ]; then
-        cat >> ${local_conf_file_name} << EOF
-disable_service $(csv2ssv "${DISABLE_OS_SERVICES}")
-EOF
-    fi
+    add_os_services "${CORE_OS_CONTROL_SERVICES}" "${ENABLE_OS_SERVICES}" "${DISABLE_OS_SERVICES}" "${local_conf_file_name}"
 
     cat >> ${local_conf_file_name} << EOF
 
@@ -101,9 +155,6 @@ HOST_IP=${HOSTIP}
 SERVICE_HOST=\$HOST_IP
 Q_ML2_TENANT_NETWORK_TYPE=${TENANT_NETWORK_TYPE}
 NEUTRON_CREATE_INITIAL_NETWORKS=${CREATE_INITIAL_NETWORKS}
-#Q_PLUGIN=ml2
-#ENABLE_TENANT_TUNNELS=True
-#LIBVIRT_TYPE=qemu
 
 ODL_MODE=manual
 ODL_MGR_IP=${MGRIP}
@@ -240,30 +291,13 @@ RECLONE=${RECLONE}
 disable_all_services
 EOF
 
-    CORE_OS_SERVICES="n-cpu,odl-compute"
-    cat >> ${local_conf_file_name} << EOF
-enable_service $(csv2ssv "${CORE_OS_SERVICES}")
-EOF
-    if [ -n "${ENABLE_OS_COMPUTE_SERVICES}" ]; then
-        cat >> ${local_conf_file_name} << EOF
-enable_service $(csv2ssv "${ENABLE_OS_COMPUTE_SERVICES}")
-EOF
-    fi
-    if [ -n "${DISABLE_OS_SERVICES}" ]; then
-        cat >> ${local_conf_file_name} << EOF
-disable_service $(csv2ssv "${DISABLE_OS_SERVICES}")
-EOF
-    fi
+    add_os_services "${CORE_OS_COMPUTE_SERVICES}" "${ENABLE_OS_COMPUTE_SERVICES}" "${DISABLE_OS_SERVICES}" "${local_conf_file_name}"
 
     cat >> ${local_conf_file_name} << EOF
 
 HOST_IP=${HOSTIP}
 SERVICE_HOST=${SERVICEHOST}
 Q_ML2_TENANT_NETWORK_TYPE=${TENANT_NETWORK_TYPE}
-#Q_PLUGIN=ml2
-#ENABLE_TENANT_TUNNELS=True
-#LIBVIRT_TYPE=qemu
-#MULTI_HOST=1
 
 ODL_MODE=manual
 ODL_MGR_IP=${MGRIP}
@@ -394,8 +428,22 @@ EOF
     ${SSH} ${MGRIP} "sudo bash /tmp/deploy_ha_proxy.sh"
 } # configure_haproxy_for_neutron_requests()
 
+# Collect the list of files on the hosts
+function list_files () {
+    local ip=$1
+    local folder=$2
+    ${SSH} ${ip} "sudo find /etc > /tmp/find.etc.txt"
+    ${SSH} ${ip} "sudo find /opt/stack > /tmp/find.opt.stack.txt"
+    ${SSH} ${ip} "sudo find /var > /tmp/find2.txt"
+    rsync --rsync-path="sudo rsync" -arv --list-only ssh ${ip}:/var/ > ${folder}/rsync.var.txt
+    scp ${ip}:/tmp/find.etc.txt ${folder}
+    scp ${ip}:/tmp/find.opt.stack.txt ${folder}
+    scp ${ip}:/tmp/find2.txt ${folder}
+}
+
 function collect_logs () {
     set +e  # We do not want to create red dot just because something went wrong while fetching logs.
+
     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
         CONTROLLERIP=ODL_SYSTEM_${i}_IP
         echo "Lets's take the karaf thread dump again..."
@@ -417,7 +465,7 @@ sudo netstat -punta
 echo -e "\nsudo getenforce\n"
 sudo getenforce
 echo -e "\njournalctl > /tmp/journalctl.log\n"
-journalctl > /tmp/journalctl.log
+sudo journalctl > /tmp/journalctl.log
 EOF
 
     sleep 5
@@ -431,7 +479,9 @@ EOF
         scp "${!CONTROLLERIP}:/tmp/odl${i}_zrpcd.log.tar" "${WORKSPACE}/odl${i}_zrpcd.log.tar"
         tar -xvf ${WORKSPACE}/odl${i}_karaf.log.tar -C . --strip-components 2 --transform s/karaf/odl${i}_karaf/g
         grep "ROBOT MESSAGE\| ERROR " odl${i}_karaf.log > odl${i}_err.log
-        grep "ROBOT MESSAGE\|Exception" odl${i}_karaf.log > odl${i}_exception.log
+        # Print ROBOT lines, Print Caused by...Exception: lines,
+        # Print Exception{ lines as well as the previous line that has the timestamp for context
+        sed -n -e '/ROBOT MESSAGE/P' -e '/Caused by.*Exception:/P' -e '$!N;/Exception{/P;D' odl${i}_karaf.log > odl${i}_exception.log
         grep "ROBOT MESSAGE\| ERROR \| WARN \|Exception" odl${i}_karaf.log > odl${i}_err_warn_exception.log
         rm ${WORKSPACE}/odl${i}_karaf.log.tar
     done
@@ -441,72 +491,78 @@ EOF
     # archives build step will essentially be a noop.
     mkdir -p ${WORKSPACE}/archives
 
+    print_job_parameters > ${WORKSPACE}/archives/params.txt
+
     # Control Node
     for i in `seq 1 ${NUM_OPENSTACK_CONTROL_NODES}`; do
-        OS_CTRL_IP=OPENSTACK_CONTROL_NODE_${i}_IP
-        OS_CTRL_FOLDER="control_${i}"
-        mkdir -p ${OS_CTRL_FOLDER}
-        scp ${!OS_CTRL_IP}:/opt/stack/devstack/nohup.out ${OS_CTRL_FOLDER}/stack.log
-        scp ${!OS_CTRL_IP}:/var/log/openvswitch/ovs-vswitchd.log ${OS_CTRL_FOLDER}/ovs-vswitchd.log
-        scp ${!OS_CTRL_IP}:/etc/neutron/plugins/ml2/ml2_conf.ini ${OS_CTRL_FOLDER}
-        scp ${!OS_CTRL_IP}:/etc/neutron/dhcp_agent.ini ${OS_CTRL_FOLDER}
-        scp ${!OS_CTRL_IP}:/etc/neutron/neutron.conf ${OS_CTRL_FOLDER}/neutron.conf
-        scp ${!OS_CTRL_IP}:/etc/nova/nova.conf ${OS_CTRL_FOLDER}/nova.conf
-        scp ${!OS_CTRL_IP}:/etc/kuryr/kuryr.conf ${OS_CTRL_FOLDER}/kuryr.conf
-        scp ${!OS_CTRL_IP}:/etc/neutron/neutron_lbaas.conf ${OS_CTRL_FOLDER}/neutron-lbaas.conf
-        scp ${!OS_CTRL_IP}:/etc/neutron/services/loadbalancer/haproxy/lbaas_agent.ini ${OS_CTRL_FOLDER}/lbaas-agent.ini
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OS_CTRL_IP}:/usr/lib/systemd/system/haproxy.service ${OS_CTRL_FOLDER}
-        rsync -avhe ssh ${!OS_CTRL_IP}:/opt/stack/logs/* ${OS_CTRL_FOLDER} # rsync to prevent copying of symbolic links
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OS_CTRL_IP}:/etc/hosts ${OS_CTRL_FOLDER}/hosts.log
-        # Use rsync with sudo to get access to the log dir.
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OS_CTRL_IP}:/var/log/audit/audit.log ${OS_CTRL_FOLDER}
-        scp ${!OS_CTRL_IP}:/tmp/get_devstack.sh.txt ${OS_CTRL_FOLDER}
-        scp extra_debug.sh ${!OS_CTRL_IP}:/tmp
-        ${SSH} ${!OS_CTRL_IP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log"
-        scp ${!OS_CTRL_IP}:/tmp/extra_debug.log ${OS_CTRL_FOLDER}/extra_debug.log
-        scp ${!OS_CTRL_IP}:/tmp/journalctl.log ${OS_CTRL_FOLDER}
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OS_CTRL_IP}:/var/log/messages ${OS_CTRL_FOLDER}
-        scp ${!OS_CTRL_IP}:/tmp/*.xz ${OS_CTRL_FOLDER}/
-        mv local.conf_control_${!OS_CTRL_IP} ${OS_CTRL_FOLDER}/local.conf
-        mv ${OS_CTRL_FOLDER} ${WORKSPACE}/archives/
+        OSIP=OPENSTACK_CONTROL_NODE_${i}_IP
+        NODE_FOLDER="control_${i}"
+        mkdir -p ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/kuryr/kuryr.conf ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/neutron/dhcp_agent.ini ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/neutron/metadata_agent.ini ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/neutron/neutron.conf ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/neutron/neutron_lbaas.conf ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/neutron/plugins/ml2/ml2_conf.ini ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/neutron/services/loadbalancer/haproxy/lbaas_agent.ini ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/nova/nova.conf ${NODE_FOLDER}
+        scp ${!OSIP}:/opt/stack/devstack/nohup.out ${NODE_FOLDER}/stack.log
+        scp ${!OSIP}:/opt/stack/requirements/upper-constraints.txt ${NODE_FOLDER}
+        scp ${!OSIP}:/opt/stack/tempest/etc/tempest.conf ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/get_devstack.sh.txt ${NODE_FOLDER}
+        scp ${!OSIP}:/var/log/openvswitch/ovs-vswitchd.log ${NODE_FOLDER}
+        scp ${!OSIP}:/var/log/openvswitch/ovsdb-server.log ${NODE_FOLDER}
+        list_files "${!OSIP}" "${NODE_FOLDER}"
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/etc/hosts ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/usr/lib/systemd/system/haproxy.service ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/audit/audit.log ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/messages ${NODE_FOLDER}
+        rsync -avhe ssh ${!OSIP}:/opt/stack/logs/* ${NODE_FOLDER} # rsync to prevent copying of symbolic links
+        scp extra_debug.sh ${!OSIP}:/tmp
+        ${SSH} ${!OSIP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log"
+        scp ${!OSIP}:/tmp/extra_debug.log ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/journalctl.log ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/*.xz ${NODE_FOLDER}
+        mv local.conf_control_${!OSIP} ${NODE_FOLDER}/local.conf
+        mv ${NODE_FOLDER} ${WORKSPACE}/archives/
     done
 
     # Compute Nodes
     for i in `seq 1 ${NUM_OPENSTACK_COMPUTE_NODES}`; do
         OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP
-        OS_COMPUTE_FOLDER="compute_${i}"
-        mkdir -p ${OS_COMPUTE_FOLDER}
-        scp ${!OSIP}:/opt/stack/devstack/nohup.out ${OS_COMPUTE_FOLDER}/stack.log
-        scp ${!OSIP}:/var/log/openvswitch/ovs-vswitchd.log ${OS_COMPUTE_FOLDER}/ovs-vswitchd.log
-        scp ${!OSIP}:/var/log/openvswitch/ovsdb-server.log ${OS_COMPUTE_FOLDER}/ovsdb-server.log
-        scp ${!OSIP}:/var/log/libvirt/libvirtd.log* ${OS_COMPUTE_FOLDER}
-        scp ${!OSIP}:/var/log/libvirt/qeum/*.log ${OS_COMPUTE_FOLDER}
-        scp ${!OSIP}:/etc/nova/nova.conf ${OS_COMPUTE_FOLDER}/nova.conf
-        rsync -avhe ssh ${!OSIP}:/opt/stack/logs/* ${OS_COMPUTE_FOLDER} # rsync to prevent copying of symbolic links
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/etc/hosts ${OS_COMPUTE_FOLDER}/hosts.log
-        # Use rsync with sudo to get access to the log dir. Also can't use wildcard because the dirs only have
-        # exec permissions which doesn't allow ls.
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/libvirt ${OS_COMPUTE_FOLDER}
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/audit/audit.log ${OS_COMPUTE_FOLDER}
-        scp ${!OSIP}:/tmp/get_devstack.sh.txt ${OS_COMPUTE_FOLDER}
+        NODE_FOLDER="compute_${i}"
+        mkdir -p ${NODE_FOLDER}
+        scp ${!OSIP}:/etc/nova/nova.conf ${NODE_FOLDER}
+        scp ${!OSIP}:/opt/stack/devstack/nohup.out ${NODE_FOLDER}/stack.log
+        scp ${!OSIP}:/opt/stack/requirements/upper-constraints.txt ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/get_devstack.sh.txt ${NODE_FOLDER}
+        scp ${!OSIP}:/var/log/openvswitch/ovs-vswitchd.log ${NODE_FOLDER}
+        scp ${!OSIP}:/var/log/openvswitch/ovsdb-server.log ${NODE_FOLDER}
+        list_files "${!OSIP}" "${NODE_FOLDER}"
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/etc/hosts ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/audit/audit.log ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/dmesg.log ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/libvirt ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/messages ${NODE_FOLDER}
+        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/nova-agent.log ${NODE_FOLDER}
+        rsync -avhe ssh ${!OSIP}:/opt/stack/logs/* ${NODE_FOLDER} # rsync to prevent copying of symbolic links
         scp extra_debug.sh ${!OSIP}:/tmp
         ${SSH} ${!OSIP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log"
-        scp ${!OSIP}:/tmp/extra_debug.log ${OS_COMPUTE_FOLDER}/extra_debug.log
-        scp ${!OSIP}:/tmp/journalctl.log ${OS_COMPUTE_FOLDER}
-        rsync --rsync-path="sudo rsync" -avhe ssh ${!OSIP}:/var/log/messages ${OS_COMPUTE_FOLDER}
-        scp ${!OSIP}:/tmp/*.xz ${OS_COMPUTE_FOLDER}/
-        mv local.conf_compute_${!OSIP} ${OS_COMPUTE_FOLDER}/local.conf
-        mv ${OS_COMPUTE_FOLDER} ${WORKSPACE}/archives/
+        scp ${!OSIP}:/tmp/extra_debug.log ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/journalctl.log ${NODE_FOLDER}
+        scp ${!OSIP}:/tmp/*.xz ${NODE_FOLDER}/
+        mv local.conf_compute_${!OSIP} ${NODE_FOLDER}/local.conf
+        mv ${NODE_FOLDER} ${WORKSPACE}/archives/
     done
 
     # Tempest
     DEVSTACK_TEMPEST_DIR="/opt/stack/tempest"
     TESTREPO=".stestr"
+    TEMPEST_LOGS_DIR=${WORKSPACE}/archives/tempest
     # Look for tempest test results in the $TESTREPO dir and copy if found
     if ${SSH} ${OPENSTACK_CONTROL_NODE_1_IP} "sudo sh -c '[ -f ${DEVSTACK_TEMPEST_DIR}/${TESTREPO}/0 ]'"; then
         ${SSH} ${OPENSTACK_CONTROL_NODE_1_IP} "for I in \$(sudo ls ${DEVSTACK_TEMPEST_DIR}/${TESTREPO}/ | grep -E '^[0-9]+$'); do sudo sh -c \"${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/subunit-1to2 < ${DEVSTACK_TEMPEST_DIR}/${TESTREPO}/\${I} >> ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt\"; done"
         ${SSH} ${OPENSTACK_CONTROL_NODE_1_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/python ${DEVSTACK_TEMPEST_DIR}/.tox/tempest/lib/python2.7/site-packages/os_testr/subunit2html.py ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt ${DEVSTACK_TEMPEST_DIR}/tempest_results.html'"
-        TEMPEST_LOGS_DIR=${WORKSPACE}/archives/tempest
         mkdir -p ${TEMPEST_LOGS_DIR}
         scp ${OPENSTACK_CONTROL_NODE_1_IP}:${DEVSTACK_TEMPEST_DIR}/tempest_results.html ${TEMPEST_LOGS_DIR}
         scp ${OPENSTACK_CONTROL_NODE_1_IP}:${DEVSTACK_TEMPEST_DIR}/tempest.log ${TEMPEST_LOGS_DIR}
@@ -516,12 +572,6 @@ EOF
     fi
 } # collect_logs()
 
-if [ "${ODL_ML2_BRANCH}" != "stable/ocata" ]; then
-   RECLONE=no
-else
-   RECLONE=yes
-fi
-
 # if we are using the new netvirt impl, as determined by the feature name
 # odl-netvirt-openstack (note: old impl is odl-ovsdb-openstack) then we
 # want PROVIDER_MAPPINGS to be used -- this should be fixed if we want to support
@@ -541,6 +591,29 @@ else
     ODL_L3=False
 fi
 
+RECLONE=False
+
+# Always compare the lists below against the devstack upstream ENABLED_SERVICES in
+# https://github.com/openstack-dev/devstack/blob/master/stackrc#L52
+# ODL CSIT does not use vnc, cinder, q-agt, q-l3 or horizon so they are not included below.
+# collect performance stats
+CORE_OS_CONTROL_SERVICES="dstat"
+# Glance
+CORE_OS_CONTROL_SERVICES+=",g-api,g-reg"
+# Keystone
+CORE_OS_CONTROL_SERVICES+=",key"
+# Nova - services to support libvirt
+CORE_OS_CONTROL_SERVICES+=",n-api,n-api-meta,n-cauth,n-cond,n-crt,n-obj,n-sch"
+# ODL - services to connect to ODL
+CORE_OS_CONTROL_SERVICES+=",odl-compute,odl-neutron"
+# Neutron
+CORE_OS_CONTROL_SERVICES+=",q-dhcp,q-meta,q-svc"
+# Additional services
+CORE_OS_CONTROL_SERVICES+=",mysql,rabbit"
+
+# computes only need nova and odl
+CORE_OS_COMPUTE_SERVICES="n-cpu,odl-compute"
+
 cat > ${WORKSPACE}/disable_firewall.sh << EOF
 sudo systemctl stop firewalld
 sudo systemctl stop iptables
@@ -642,6 +715,17 @@ for i in `seq 1 ${NUM_OPENSTACK_COMPUTE_NODES}`; do
     ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
     ssh ${!COMPUTEIP} "ps -ef | grep stack.sh"
     os_node_list+=(${!COMPUTEIP})
+    # Workaround for https://review.openstack.org/#/c/491032/
+    # Modify upper-constraints to use libvirt-python 3.2.0
+    if [ "${ODL_ML2_BRANCH}" == "stable/ocata" ]; then
+        ${SSH} ${!COMPUTEIP} "
+            cd /opt/stack;
+            git clone https://git.openstack.org/openstack/requirements;
+            cd requirements;
+            git checkout stable/ocata;
+            sed -i s/libvirt-python===2.5.0/libvirt-python===3.2.0/ upper-constraints.txt
+        "
+    fi
 done
 
 echo "nodelist: ${os_node_list[*]}"
@@ -725,8 +809,8 @@ for i in `seq 1 ${NUM_OPENSTACK_SITES}`; do
 
     echo "sleep for 60s and print hypervisor-list"
     sleep 60
-    # In anything after Newton, if we do not enable the n-cpu in control node
-    # We need to discover hosts manually and ensure that they are mapped to cells.
+    # In Ocata if we do not enable the n-cpu in control node then
+    # we need to discover hosts manually and ensure that they are mapped to cells.
     # reference: https://ask.openstack.org/en/question/102256/how-to-configure-placement-service-for-compute-node-on-ocata/
     if [ "${OPENSTACK_BRANCH}" == "stable/ocata" ]; then
         scp ${WORKSPACE}/setup_host_cell_mapping.sh  ${!CONTROLIP}:/tmp
@@ -877,7 +961,7 @@ if [ -z "${SUITES}" ]; then
     SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
 fi
 
-if [ "${OPENSTACK_BRANCH}" == "stable/pike" ]; then
+if [ "${OPENSTACK_BRANCH}" == "stable/pike" ] || [ "${OPENSTACK_BRANCH}" == "master" ]; then
    AUTH="http://${!CONTROLIP}/identity"
 else
    AUTH="http://${!CONTROLIP}:35357/v3"