X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Fintegration%2Finclude-raw-integration-deploy-openstack-run-test.sh;h=703e8f6aef414b0582a165e22b4fe25c292b358f;hb=refs%2Fchanges%2F69%2F52869%2F7;hp=38c520b1559895568dac083955fedae097300150;hpb=d9b4a817292e1229143a4e9c918760e16870d0b9;p=releng%2Fbuilder.git

diff --git a/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh b/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh
index 38c520b15..703e8f6ae 100644
--- a/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh
+++ b/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh
@@ -4,6 +4,13 @@
 # script.
 source ${ROBOT_VENV}/bin/activate
 
+# TODO: remove this work to run changes.py if/when it's moved higher up to be visible at the Robot level
+echo "showing recent changes that made it in to the distribution used by this job"
+pip install --upgrade urllib3
+python ${WORKSPACE}/test/tools/distchanges/changes.py -d /tmp/distribution_folder \
+                  -u ${ACTUALBUNDLEURL} -b ${DISTROBRANCH} \
+                  -r ssh://jenkins-${SILO}@git.opendaylight.org:29418 || true
+
 echo "#################################################"
 echo "##         Deploy Openstack 3-node             ##"
 echo "#################################################"
@@ -14,7 +21,7 @@ SSH="ssh -t -t"
 function create_control_node_local_conf {
 local_conf_file_name=${WORKSPACE}/local.conf_control
 #Needs to be removed
-if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
+if [ "${ODL_ML2_BRANCH}" != "stable/ocata" ]; then
    RECLONE=no
 else
    RECLONE=yes
@@ -40,13 +47,33 @@ cat >> ${local_conf_file_name} << EOF
 enable_service ${service_name}
 EOF
 done
+for plugin_name in ${ENABLE_OS_PLUGINS}
+do
+if [ "$plugin_name" == "networking-odl" ]; then
+    ENABLE_PLUGIN_ARGS="${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}"
+elif [ "$plugin_name" == "kuryr-kubernetes" ]; then
+    ENABLE_PLUGIN_ARGS="${DEVSTACK_KUBERNETES_PLUGIN_REPO} master" # note: kuryr-kubernetes only exists in master at the moment
+elif [ "$plugin_name" == "neutron-lbaas" ]; then
+    ENABLE_PLUGIN_ARGS="${DEVSTACK_LBAAS_PLUGIN_REPO} ${OPENSTACK_BRANCH}"
+else
+    echo "Error: Invalid plugin $plugin_name, unsupported"
+    continue
+fi
+cat >> ${local_conf_file_name} << EOF
+enable_plugin ${plugin_name} ${ENABLE_PLUGIN_ARGS}
+EOF
+done
 unset IFS
-
+if [ "${OPENSTACK_BRANCH}" == "master" ] || [ "${OPENSTACK_BRANCH}" == "stable/ocata" ]; then # Ocata+
+    # placement is mandatory for nova since Ocata, note that this requires computes to enable placement-client
+    # this should be moved into enabled_services for each job (but only for Ocata)
+    echo "enable_service placement-api" >> ${local_conf_file_name}
+fi
 cat >> ${local_conf_file_name} << EOF
 HOST_IP=$OPENSTACK_CONTROL_NODE_IP
 SERVICE_HOST=\$HOST_IP
 
-NEUTRON_CREATE_INITIAL_NETWORKS=False
+NEUTRON_CREATE_INITIAL_NETWORKS=${CREATE_INITIAL_NETWORKS}
 Q_PLUGIN=ml2
 Q_ML2_TENANT_NETWORK_TYPE=${TENANT_NETWORK_TYPE}
 Q_OVS_USE_VETH=True
@@ -65,14 +92,24 @@ SERVICE_TOKEN=service
 SERVICE_PASSWORD=admin
 ADMIN_PASSWORD=admin
 
-enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
-
 ODL_PORT=8080
 ODL_MODE=externalodl
+ODL_PORT_BINDING_CONTROLLER=${ODL_ML2_PORT_BINDING}
+
 LIBVIRT_TYPE=qemu
 
+NEUTRON_LBAAS_SERVICE_PROVIDERV2=${LBAAS_SERVICE_PROVIDER} # Only relevant if neutron-lbaas plugin is enabled
 EOF
 
+if [ "${ENABLE_NETWORKING_L2GW}" == "yes" ]; then
+cat >> ${local_conf_file_name} << EOF
+
+enable_plugin networking-l2gw ${NETWORKING_L2GW_DRIVER} ${ODL_ML2_BRANCH}
+NETWORKING_L2GW_SERVICE_DRIVER=L2GW:OpenDaylight:networking_odl.l2gateway.driver.OpenDaylightL2gwDriver:default
+ENABLED_SERVICES+=,neutron,q-svc,nova,q-meta
+
+EOF
+fi
 
 if [ "${ODL_ML2_DRIVER_VERSION}" == "v2" ]; then
     echo "ODL_V2DRIVER=True" >> ${local_conf_file_name}
@@ -124,23 +161,26 @@ fi
 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
 cat >> ${local_conf_file_name} << EOF
 PUBLIC_BRIDGE=${PUBLIC_BRIDGE}
-PUBLIC_PHYSICAL_NETWORK=physnet1 # FIXME this should be a parameter
+PUBLIC_PHYSICAL_NETWORK=${PUBLIC_PHYSICAL_NETWORK}
+ML2_VLAN_RANGES=${PUBLIC_PHYSICAL_NETWORK}
 ODL_PROVIDER_MAPPINGS=${ODL_PROVIDER_MAPPINGS}
 
 disable_service q-l3
-Q_L3_ENABLED=True
-ODL_L3=${ODL_L3}
 PUBLIC_INTERFACE=br100
 EOF
 
+if [ -z ${DISABLE_ODL_L3_PLUGIN} ] || [ "${DISABLE_ODL_L3_PLUGIN}" == "no" ]; then
 if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
 cat >> ${local_conf_file_name} << EOF
+Q_L3_ENABLED=True
+ODL_L3=${ODL_L3}
 [[post-config|\$NEUTRON_CONF]]
 [DEFAULT]
 service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
 
 EOF
 fi #check for ODL_ML2_BRANCH
+fi #check for DISABLE_ODL_L3_PLUGIN
 
 fi #ODL_ENABLE_L3_FWD check
 
@@ -149,6 +189,12 @@ cat >> ${local_conf_file_name} << EOF
 [agent]
 minimize_polling=True
 
+[ml2]
+# Needed for VLAN provider tests - because our provider networks are always encapsulated in VXLAN (br-phys1)
+# MTU(1440) + VXLAN(50) + VLAN(4) = 1494 < MTU eth0/br-phys1(1500)
+physical_network_mtus = ${PUBLIC_PHYSICAL_NETWORK}:1440
+path_mtu = 1490
+
 [[post-config|/etc/neutron/dhcp_agent.ini]]
 [DEFAULT]
 force_metadata = True
@@ -167,11 +213,18 @@ cat ${local_conf_file_name}
 function create_compute_node_local_conf {
 HOSTIP=$1
 #Needs to be removed
-if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
+if [ "${ODL_ML2_BRANCH}" != "stable/ocata" ]; then
    RECLONE=no
 else
    RECLONE=yes
 fi
+if [ "${OPENSTACK_BRANCH}" == "master" ] || [ "${OPENSTACK_BRANCH}" == "stable/ocata" ]; then # Ocata+
+    # placement is mandatory for nova since Ocata, note that this requires controller to enable placement-api
+    ENABLED_SERVICES="n-cpu,placement-client"
+else
+    ENABLED_SERVICES="n-cpu"
+fi
+
 local_conf_file_name=${WORKSPACE}/local.conf_compute_${HOSTIP}
 cat > ${local_conf_file_name} << EOF
 [[local|localrc]]
@@ -182,8 +235,7 @@ RECLONE=${RECLONE}
 
 NOVA_VNC_ENABLED=True
 MULTI_HOST=1
-ENABLED_SERVICES=n-cpu
-
+ENABLED_SERVICES=${ENABLED_SERVICES}
 HOST_IP=${HOSTIP}
 SERVICE_HOST=${OPENSTACK_CONTROL_NODE_IP}
 
@@ -204,11 +256,17 @@ SERVICE_TOKEN=service
 SERVICE_PASSWORD=admin
 ADMIN_PASSWORD=admin
 
-enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
 ODL_MODE=compute
+ODL_PORT_BINDING_CONTROLLER=${ODL_ML2_PORT_BINDING}
 LIBVIRT_TYPE=qemu
 EOF
 
+if [[ "${ENABLE_OS_PLUGINS}" =~ networking-odl ]]; then
+cat >> ${local_conf_file_name} << EOF
+enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
+EOF
+fi
+
 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
 odl_list=${ODL_SYSTEM_1_IP}
 for i in `seq 2 ${NUM_ODL_SYSTEM}`
@@ -250,7 +308,7 @@ Q_L3_ENABLED=True
 ODL_L3=${ODL_L3}
 PUBLIC_INTERFACE=br100 # FIXME do we use br100 at all?
 PUBLIC_BRIDGE=${PUBLIC_BRIDGE}
-PUBLIC_PHYSICAL_NETWORK=physnet1 # FIXME this should be a parameter
+PUBLIC_PHYSICAL_NETWORK=${PUBLIC_PHYSICAL_NETWORK}
 ODL_PROVIDER_MAPPINGS=${ODL_PROVIDER_MAPPINGS}
 EOF
 fi
@@ -358,6 +416,9 @@ do
     ${SSH} "${!CONTROLLERIP}"  "tar -cf /tmp/odl${i}_karaf.log.tar /tmp/odl_log/*"
     scp "${!CONTROLLERIP}:/tmp/odl${i}_karaf.log.tar" "${WORKSPACE}/odl${i}_karaf.log.tar"
     tar -xvf ${WORKSPACE}/odl${i}_karaf.log.tar -C . --strip-components 2 --transform s/karaf/odl${i}_karaf/g
+    grep "ROBOT MESSAGE\| ERROR " odl${i}_karaf.log > odl${i}_err.log
+    grep "ROBOT MESSAGE\|Exception" odl${i}_karaf.log > odl${i}_exception.log
+    grep "ROBOT MESSAGE\| ERROR \| WARN \|Exception" odl${i}_karaf.log > odl${i}_err_warn_exception.log
     rm ${WORKSPACE}/odl${i}_karaf.log.tar
 done
 
@@ -371,6 +432,7 @@ OS_CTRL_FOLDER="control"
 mkdir -p ${OS_CTRL_FOLDER}
 scp ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/nohup.out ${OS_CTRL_FOLDER}/stack.log
 scp ${OPENSTACK_CONTROL_NODE_IP}:/var/log/openvswitch/ovs-vswitchd.log ${OS_CTRL_FOLDER}/ovs-vswitchd.log
+scp ${OPENSTACK_CONTROL_NODE_IP}:/etc/neutron/neutron.conf ${OS_CTRL_FOLDER}/neutron.conf
 rsync -avhe ssh ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/logs/* ${OS_CTRL_FOLDER} # rsync to prevent copying of symbolic links
 scp extra_debug.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp
 ${SSH} ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log"
@@ -386,6 +448,7 @@ do
     mkdir -p ${OS_COMPUTE_FOLDER}
     scp ${!OSIP}:/opt/stack/devstack/nohup.out ${OS_COMPUTE_FOLDER}/stack.log
     scp ${!OSIP}:/var/log/openvswitch/ovs-vswitchd.log ${OS_COMPUTE_FOLDER}/ovs-vswitchd.log
+    scp ${!OSIP}:/etc/nova/nova.conf ${OS_COMPUTE_FOLDER}/nova.conf
     rsync -avhe ssh ${!OSIP}:/opt/stack/logs/* ${OS_COMPUTE_FOLDER} # rsync to prevent copying of symbolic links
     scp extra_debug.sh ${!OSIP}:/tmp
     ${SSH} ${!OSIP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log"
@@ -395,6 +458,18 @@ do
 done
 
 ls local.conf* | xargs -I % mv % %.log
+
+# Tempest
+DEVSTACK_TEMPEST_DIR="/opt/stack/tempest"
+if $(ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '[ -f ${DEVSTACK_TEMPEST_DIR}/.testrepository/0 ]'"); then # if Tempest results exist
+    ssh ${OPENSTACK_CONTROL_NODE_IP} "for I in \$(sudo ls ${DEVSTACK_TEMPEST_DIR}/.testrepository/ | grep -E '^[0-9]+$'); do sudo sh -c \"${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/subunit-1to2 < ${DEVSTACK_TEMPEST_DIR}/.testrepository/\${I} >> ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt\"; done"
+    ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/python ${DEVSTACK_TEMPEST_DIR}/.tox/tempest/lib/python2.7/site-packages/os_testr/subunit2html.py ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt ${DEVSTACK_TEMPEST_DIR}/tempest_results.html'"
+    TEMPEST_LOGS_DIR=${WORKSPACE}/archives/tempest
+    mkdir -p ${TEMPEST_LOGS_DIR}
+    scp ${OPENSTACK_CONTROL_NODE_IP}:${DEVSTACK_TEMPEST_DIR}/tempest_results.html ${TEMPEST_LOGS_DIR}
+    scp ${OPENSTACK_CONTROL_NODE_IP}:${DEVSTACK_TEMPEST_DIR}/tempest.log ${TEMPEST_LOGS_DIR}
+    mv ${WORKSPACE}/tempest_output* ${TEMPEST_LOGS_DIR}
+fi
 }
 
 cat > ${WORKSPACE}/disable_firewall.sh << EOF
@@ -438,9 +513,31 @@ ${SSH} ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/get_devstack.sh"
 create_control_node_local_conf
 scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf
 
+cat > "${WORKSPACE}/manual_install_package.sh" << EOF
+cd /opt/stack
+git clone "\$1"
+cd "\$2"
+git checkout "\$3"
+sudo python setup.py install
+
+EOF
+
+
+# Workworund for successful stacking with  Mitaka
 if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
-ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt"
-ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://github.com/openstack/python-openstacksdk; cd python-openstacksdk; sudo python setup.py install"
+
+  # Workaround for problems with latest versions/specified versions in requirements of openstack
+  # Openstacksdk,libvirt-python -> the current version does not work with  Mitaka diue to some requirements
+  # conflict and breaks when trying to stack
+  # paramiko -> Problems with tempest tests due to paramiko incompatibility with pycrypto.
+  # the problem has been solved with  version 1.17. If the latest version of paramiko is used, it causes
+  # other timeout problems
+  ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt; sed -i /paramiko/d upper-constraints.txt"
+  scp "${WORKSPACE}/manual_install_package.sh"  "${OPENSTACK_CONTROL_NODE_IP}:/tmp"
+  ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install deprecation"
+  # Fix for recent requirements update  in the  master branch of the sdk.The section must be replaced with a better fix.
+  ssh "${OPENSTACK_CONTROL_NODE_IP}" "sh /tmp/manual_install_package.sh https://github.com/openstack/python-openstacksdk python-openstacksdk 0.9.14"
+  ssh "${OPENSTACK_CONTROL_NODE_IP}" "sh /tmp/manual_install_package.sh https://github.com/paramiko/paramiko paramiko 1.17"
 fi
 
 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
@@ -448,6 +545,10 @@ ssh ${OPENSTACK_CONTROL_NODE_IP} "ps -ef | grep stack.sh"
 ssh ${OPENSTACK_CONTROL_NODE_IP} "ls -lrt /opt/stack/devstack/nohup.out"
 os_node_list+=(${OPENSTACK_CONTROL_NODE_IP})
 
+#Workaround for stable/newton jobs
+if [ "${ODL_ML2_BRANCH}" == "stable/newton" ]; then
+  ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/newton; sed -i /appdirs/d upper-constraints.txt"
+fi
 
 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 do
@@ -521,7 +622,15 @@ ${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo bash /tmp/disable_firewall.sh"
 echo "sleep for a minute and print hypervisor-list"
 sleep 60
 ${SSH} ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; source openrc admin admin; nova hypervisor-list"
-expected_num_hypervisors=$((NUM_OPENSTACK_SYSTEM - 1))
+# in the case that we are doing openstack (control + compute) all in one node, then the number of hypervisors
+# will be the same as the number of openstack systems. However, if we are doing multinode openstack then the
+# assumption is we have a single control node and the rest are compute nodes, so the number of expected hypervisors
+# is one less than the total number of openstack systems
+if [ "${NUM_OPENSTACK_SYSTEM}" -eq 1 ]; then
+  expected_num_hypervisors=1
+else
+  expected_num_hypervisors=$((NUM_OPENSTACK_SYSTEM - 1))
+fi
 num_hypervisors=$(${SSH} ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; source openrc admin admin; openstack hypervisor list -f value | wc -l" | tail -1 | tr -d "\r")
 if ! [ "${num_hypervisors}" ] || ! [ ${num_hypervisors} -eq ${expected_num_hypervisors} ]; then
   echo "Error: Only $num_hypervisors hypervisors detected, expected $expected_num_hypervisors"
@@ -560,9 +669,43 @@ do
     ${SSH} $ip "sudo ovs-vsctl --may-exist add-br $PUBLIC_BRIDGE -- set bridge $PUBLIC_BRIDGE other-config:disable-in-band=true other_config:hwaddr=f6:00:00:ff:01:0$((devstack_index++))"
 done
 
+# ipsec support
+if [ "${IPSEC_VXLAN_TUNNELS_ENABLED}" == "yes" ]; then
+    ALL_NODES=(${OPENSTACK_CONTROL_NODE_IP} ${COMPUTE_IPS[*]})
+    for ((inx_ip1=0; inx_ip1<$((${#ALL_NODES[@]} - 1)); inx_ip1++))
+    do
+        for ((inx_ip2=$((inx_ip1 + 1)); inx_ip2<${#ALL_NODES[@]}; inx_ip2++))
+        do
+            KEY1=0x$(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64)
+            KEY2=0x$(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64)
+            ID=0x$(dd if=/dev/urandom count=4 bs=1 2> /dev/null| xxd -p -c 8)
+            ip1=${ALL_NODES[$inx_ip1]}
+            ip2=${ALL_NODES[$inx_ip2]}
+            ${SSH} $ip1 "sudo ip xfrm state add src $ip1 dst $ip2 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2"
+            ${SSH} $ip1 "sudo ip xfrm state add src $ip2 dst $ip1 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2"
+            ${SSH} $ip1 "sudo ip xfrm policy add src $ip1 dst $ip2 proto udp dir out tmpl src $ip1 dst $ip2 proto esp reqid $ID mode transport"
+            ${SSH} $ip1 "sudo ip xfrm policy add src $ip2 dst $ip1 proto udp dir in tmpl src $ip2 dst $ip1 proto esp reqid $ID mode transport"
+
+            ${SSH} $ip2 "sudo ip xfrm state add src $ip2 dst $ip1 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2"
+            ${SSH} $ip2 "sudo ip xfrm state add src $ip1 dst $ip2 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2"
+            ${SSH} $ip2 "sudo ip xfrm policy add src $ip2 dst $ip1 proto udp dir out tmpl src $ip2 dst $ip1 proto esp reqid $ID mode transport"
+            ${SSH} $ip2 "sudo ip xfrm policy add src $ip1 dst $ip2 proto udp dir in tmpl src $ip1 dst $ip2 proto esp reqid $ID mode transport"
+        done
+    done
+
+    for ip in ${OPENSTACK_CONTROL_NODE_IP} ${COMPUTE_IPS[*]}
+    do
+        echo "ip xfrm configuration for node $ip:"
+        ${SSH} $ip "sudo ip xfrm policy list"
+        ${SSH} $ip "sudo ip xfrm state list"
+    done
+fi
+
 # Control Node - PUBLIC_BRIDGE will act as the external router
 GATEWAY_IP="10.10.10.250" # FIXME this should be a parameter, also shared with integration-test
-${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ifconfig $PUBLIC_BRIDGE up ${GATEWAY_IP}/24"
+${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ip link add link ${PUBLIC_BRIDGE} name  ${PUBLIC_BRIDGE}.167 type vlan id 167"
+${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ifconfig  ${PUBLIC_BRIDGE} up"
+${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ifconfig   ${PUBLIC_BRIDGE}.167 up ${GATEWAY_IP}/24"
 compute_index=1
 for compute_ip in ${COMPUTE_IPS[*]}
 do
@@ -595,22 +738,42 @@ cat testplan.txt
 SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
 
 echo "Starting Robot test suites ${SUITES} ..."
-pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp \
--v BUNDLE_URL:${ACTUALBUNDLEURL} -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} -v JDKVERSION:${JDKVERSION} -v ODL_STREAM:${DISTROSTREAM} \
--v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} -v ODL_SYSTEM_2_IP:${ODL_SYSTEM_2_IP} \
--v ODL_SYSTEM_3_IP:${ODL_SYSTEM_3_IP} -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} -v CONTROLLER_USER:${USER} -v OS_USER:${USER} \
--v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \
--v OS_COMPUTE_1_IP:${OPENSTACK_COMPUTE_NODE_1_IP} -v OS_COMPUTE_2_IP:${OPENSTACK_COMPUTE_NODE_2_IP} \
--v HA_PROXY_IP:${HA_PROXY_IP} \
--v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack -v USER_HOME:${HOME} ${TESTOPTIONS} ${SUITES} || true
+# please add pybot -v arguments on a single line and alphabetized
+pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude \
+    -v BUNDLEFOLDER:${BUNDLEFOLDER} \
+    -v BUNDLE_URL:${ACTUALBUNDLEURL} \
+    -v CONTROLLER_USER:${USER} \
+    -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack \
+    -v HA_PROXY_IP:${HA_PROXY_IP} \
+    -v JDKVERSION:${JDKVERSION} \
+    -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} \
+    -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} \
+    -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} \
+    -v NUM_TOOLS_SYSTEM:${NUM_TOOLS_SYSTEM} \
+    -v ODL_STREAM:${DISTROSTREAM} \
+    -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} \
+    -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} \
+    -v ODL_SYSTEM_2_IP:${ODL_SYSTEM_2_IP} \
+    -v ODL_SYSTEM_3_IP:${ODL_SYSTEM_3_IP} \
+    -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \
+    -v OPENSTACK_BRANCH:${OPENSTACK_BRANCH} \
+    -v OS_COMPUTE_1_IP:${OPENSTACK_COMPUTE_NODE_1_IP} \
+    -v OS_COMPUTE_2_IP:${OPENSTACK_COMPUTE_NODE_2_IP} \
+    -v OS_USER:${USER} \
+    -v PUBLIC_PHYSICAL_NETWORK:${PUBLIC_PHYSICAL_NETWORK} \
+    -v SECURITY_GROUP_MODE:${SECURITY_GROUP_MODE} \
+    -v TOOLS_SYSTEM_IP:${TOOLS_SYSTEM_1_IP} \
+    -v TOOLS_SYSTEM_1_IP:${TOOLS_SYSTEM_1_IP} \
+    -v TOOLS_SYSTEM_2_IP:${TOOLS_SYSTEM_2_IP} \
+    -v USER_HOME:${HOME} \
+    -v WORKSPACE:/tmp \
+    ${TESTOPTIONS} ${SUITES} || true
+
+echo "Examining the files in data/log and checking filesize"
+ssh ${ODL_SYSTEM_IP} "ls -altr /tmp/${BUNDLEFOLDER}/data/log/"
+ssh ${ODL_SYSTEM_IP} "du -hs /tmp/${BUNDLEFOLDER}/data/log/*"
 
 echo "Tests Executed"
-DEVSTACK_TEMPEST_DIR="/opt/stack/tempest"
-if $(ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '[ -f ${DEVSTACK_TEMPEST_DIR}/.testrepository/0 ]'"); then # if Tempest results exist
-    ssh ${OPENSTACK_CONTROL_NODE_IP} "for I in \$(sudo ls ${DEVSTACK_TEMPEST_DIR}/.testrepository/ | grep -E '^[0-9]+$'); do sudo sh -c \"${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/subunit-1to2 < ${DEVSTACK_TEMPEST_DIR}/.testrepository/\${I} >> ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt\"; done"
-    ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/python ${DEVSTACK_TEMPEST_DIR}/.tox/tempest/lib/python2.7/site-packages/os_testr/subunit2html.py ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt ${DEVSTACK_TEMPEST_DIR}/tempest_results.html'"
-    scp ${OPENSTACK_CONTROL_NODE_IP}:${DEVSTACK_TEMPEST_DIR}/tempest_results.html ${WORKSPACE}/
-fi
 collect_logs_and_exit
 
 true  # perhaps Jenkins is testing last exit code