Merge "Add OpenStack Ocata runs for netvirt"
[releng/builder.git] / jjb / integration / include-raw-integration-deploy-openstack-run-test.sh
index f43946ee25aada95a87d031dcb9aed2d7c3c391b..a3a38778e844138469ffce1eb164b7aa7282c312 100644 (file)
@@ -8,7 +8,7 @@ source ${ROBOT_VENV}/bin/activate
 echo "showing recent changes that made it in to the distribution used by this job"
 pip install --upgrade urllib3
 python ${WORKSPACE}/test/tools/distchanges/changes.py -d /tmp/distribution_folder \
-                  -u ${ACTUALBUNDLEURL} -b ${DISTROBRANCH} \
+                  -u ${ACTUAL_BUNDLE_URL} -b ${DISTROBRANCH} \
                   -r ssh://jenkins-${SILO}@git.opendaylight.org:29418 || true
 
 echo "#################################################"
@@ -58,6 +58,8 @@ elif [ "$plugin_name" == "kuryr-kubernetes" ]; then
     ENABLE_PLUGIN_ARGS="${DEVSTACK_KUBERNETES_PLUGIN_REPO} master" # note: kuryr-kubernetes only exists in master at the moment
 elif [ "$plugin_name" == "neutron-lbaas" ]; then
     ENABLE_PLUGIN_ARGS="${DEVSTACK_LBAAS_PLUGIN_REPO} ${OPENSTACK_BRANCH}"
+elif [ "$plugin_name" == "networking-sfc" ]; then
+    ENABLE_PLUGIN_ARGS="${DEVSTACK_NETWORKING_SFC_PLUGIN_REPO} ${OPENSTACK_BRANCH}"
 else
     echo "Error: Invalid plugin $plugin_name, unsupported"
     continue
@@ -103,6 +105,8 @@ LIBVIRT_TYPE=qemu
 ODL_MGR_IP=${MGRIP}
 
 NEUTRON_LBAAS_SERVICE_PROVIDERV2=${LBAAS_SERVICE_PROVIDER} # Only relevant if neutron-lbaas plugin is enabled
+NEUTRON_SFC_DRIVERS=${ODL_SFC_DRIVER} # Only relevant if networking-sfc plugin is enabled
+NEUTRON_FLOWCLASSIFIER_DRIVERS=${ODL_SFC_DRIVER} # Only relevant if networking-sfc plugin is enabled
 EOF
 
 if [ "${ENABLE_NETWORKING_L2GW}" == "yes" ]; then
@@ -151,18 +155,20 @@ disable_service q-l3
 PUBLIC_INTERFACE=br100
 EOF
 
-if [ -z ${DISABLE_ODL_L3_PLUGIN} ] || [ "${DISABLE_ODL_L3_PLUGIN}" == "no" ]; then
-if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
+if [ "${ENABLE_NETWORKING_L2GW}" == "yes" ]; then
 cat >> ${local_conf_file_name} << EOF
-Q_L3_ENABLED=True
-ODL_L3=${ODL_L3}
 [[post-config|\$NEUTRON_CONF]]
 [DEFAULT]
-service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
+service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin, networking_l2gw.services.l2gateway.plugin.L2GatewayPlugin
 
 EOF
-fi #check for ODL_ML2_BRANCH
-fi #check for DISABLE_ODL_L3_PLUGIN
+else
+cat >> ${local_conf_file_name} << EOF
+[[post-config|\$NEUTRON_CONF]]
+[DEFAULT]
+service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
+EOF
+fi #check for ENABLE_NETWORKING_L2GW
 
 fi #ODL_ENABLE_L3_FWD check
 
@@ -172,8 +178,8 @@ cat >> ${local_conf_file_name} << EOF
 minimize_polling=True
 
 [ml2]
-# Needed for VLAN provider tests - because our provider networks are always encapsulated in VXLAN (br-phys1)
-# MTU(1440) + VXLAN(50) + VLAN(4) = 1494 < MTU eth0/br-phys1(1500)
+# Needed for VLAN provider tests - because our provider networks are always encapsulated in VXLAN (br-physnet1)
+# MTU(1440) + VXLAN(50) + VLAN(4) = 1494 < MTU eth0/br-physnet1(1500)
 physical_network_mtus = ${PUBLIC_PHYSICAL_NETWORK}:1440
 path_mtu = 1490
 
@@ -364,6 +370,9 @@ set +e  # We do not want to create red dot just because something went wrong whi
 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 do
     CONTROLLERIP=ODL_SYSTEM_${i}_IP
+    echo "Lets's take the karaf thread dump again..."
+    KARAF_PID=$(ssh ${!CONTROLLERIP} "ps aux | grep ${KARAF_ARTIFACT} | grep -v grep | tr -s ' ' | cut -f2 -d' '")
+    ssh ${!CONTROLLERIP} "jstack $KARAF_PID"> ${WORKSPACE}/karaf_${i}_threads_after.log || true
     echo "killing karaf process..."
     ${SSH} "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
 done
@@ -383,6 +392,8 @@ do
     ${SSH} "${!CONTROLLERIP}"  "cp -r /tmp/${BUNDLEFOLDER}/data/log /tmp/odl_log"
     ${SSH} "${!CONTROLLERIP}"  "tar -cf /tmp/odl${i}_karaf.log.tar /tmp/odl_log/*"
     scp "${!CONTROLLERIP}:/tmp/odl${i}_karaf.log.tar" "${WORKSPACE}/odl${i}_karaf.log.tar"
+    ${SSH} "${!CONTROLLERIP}"  "tar -cf /tmp/odl${i}_zrpcd.log.tar /tmp/zrpcd.init.log"
+    scp "${!CONTROLLERIP}:/tmp/odl${i}_zrpcd.log.tar" "${WORKSPACE}/odl${i}_zrpcd.log.tar"
     tar -xvf ${WORKSPACE}/odl${i}_karaf.log.tar -C . --strip-components 2 --transform s/karaf/odl${i}_karaf/g
     grep "ROBOT MESSAGE\| ERROR " odl${i}_karaf.log > odl${i}_err.log
     grep "ROBOT MESSAGE\|Exception" odl${i}_karaf.log > odl${i}_exception.log
@@ -475,6 +486,13 @@ cd devstack
 git checkout $OPENSTACK_BRANCH
 EOF
 
+cat > "${WORKSPACE}/setup_host_cell_mapping.sh" << EOF
+sudo nova-manage cell_v2 map_cell0
+sudo nova-manage cell_v2 simple_cell_setup
+sudo nova-manage db sync
+sudo nova-manage cell_v2 discover_hosts
+EOF
+
 [ "$NUM_OPENSTACK_SITES" ] || NUM_OPENSTACK_SITES=1
 compute_index=1
 odl_index=1
@@ -519,20 +537,6 @@ do
     ${SSH} ${!CONTROLIP} "bash /tmp/get_devstack.sh"
     create_control_node_local_conf ${!CONTROLIP} ${ODLMGRIP[$i]} "${ODL_OVS_MGRS[$i]}"
     scp ${WORKSPACE}/local.conf_control_${!CONTROLIP} ${!CONTROLIP}:/opt/stack/devstack/local.conf
-    if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
-        # Workaround for problems with latest versions/specified versions in requirements of openstack
-        # Openstacksdk,libvirt-python -> the current version does not work with  Mitaka diue to some requirements
-        # conflict and breaks when trying to stack
-        # paramiko -> Problems with tempest tests due to paramiko incompatibility with pycrypto.
-        # the problem has been solved with  version 1.17. If the latest version of paramiko is used, it causes
-        # other timeout problems
-        ssh ${!CONTROLIP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt; sed -i /paramiko/d upper-constraints.txt"
-        scp "${WORKSPACE}/manual_install_package.sh"  "${!CONTROLIP}:/tmp"
-        ssh ${!CONTROLIP} "sudo pip install deprecation"
-        # Fix for recent requirements update  in the  master branch of the sdk.The section must be replaced with a better fix.
-        ssh "${!CONTROLIP}" "sh /tmp/manual_install_package.sh https://github.com/openstack/python-openstacksdk python-openstacksdk 0.9.14"
-        ssh "${!CONTROLIP}" "sh /tmp/manual_install_package.sh https://github.com/paramiko/paramiko paramiko 1.17"
-    fi
     ssh ${!CONTROLIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
     ssh ${!CONTROLIP} "ps -ef | grep stack.sh"
     ssh ${!CONTROLIP} "ls -lrt /opt/stack/devstack/nohup.out"
@@ -555,9 +559,6 @@ do
     ${SSH} ${!COMPUTEIP} "bash /tmp/get_devstack.sh"
     create_compute_node_local_conf ${!COMPUTEIP} ${!CONTROLIP} ${ODLMGRIP[$SITE_INDEX]} "${ODL_OVS_MGRS[$SITE_INDEX]}"
     scp ${WORKSPACE}/local.conf_compute_${!COMPUTEIP} ${!COMPUTEIP}:/opt/stack/devstack/local.conf
-    if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
-        ssh ${!COMPUTEIP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /libvirt-python/d upper-constraints.txt"
-    fi
     ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
     ssh ${!COMPUTEIP} "ps -ef | grep stack.sh"
     os_node_list+=(${!COMPUTEIP})
@@ -633,7 +634,7 @@ do
     # will be the same as the number of openstack systems. However, if we are doing multinode openstack then the
     # assumption is we have a single control node and the rest are compute nodes, so the number of expected hypervisors
     # is one less than the total number of openstack systems
-    if [ $((NUM_OPENSTACK_SYSTEM / NUM_OPENSTACK_SITES - 1)) -eq 1 ]; then
+    if [ $((NUM_OPENSTACK_SYSTEM / NUM_OPENSTACK_SITES)) -eq 1 ]; then
         expected_num_hypervisors=1
     else
         expected_num_hypervisors=${NUM_COMPUTES_PER_SITE}
@@ -645,6 +646,14 @@ do
         exit 1
     fi
 
+    # For Ocata, if we do not enable the n-cpu in control node
+    # We need to discover hosts manually and ensure that they are mapped to cells.
+    # reference: https://ask.openstack.org/en/question/102256/how-to-configure-placement-service-for-compute-node-on-ocata/
+    if [ "${OPENSTACK_BRANCH}" == "stable/ocata" ]; then
+        scp ${WORKSPACE}/setup_host_cell_mapping.sh  ${!CONTROLIP}:/tmp
+        ${SSH} ${!CONTROLIP} "sudo bash /tmp/setup_host_cell_mapping.sh"
+    fi
+
     # upgrading pip, urllib3 and httplib2 so that tempest tests can be run on openstack control node
     # this needs to happen after devstack runs because it seems devstack is pulling in specific versions
     # of these libs that are not working for tempest.
@@ -711,15 +720,28 @@ do
     fi
 
     # Control Node - PUBLIC_BRIDGE will act as the external router
-    GATEWAY_IP="10.10.10.250" # FIXME this should be a parameter, also shared with integration-test
-    GATEWAY_VLAN_ID=167
-    if [[ ${CONTROLLERFEATURES} == *"odl-ovsdb-openstack"* ]]; then
-        ${SSH} ${!CONTROLIP} "sudo ifconfig ${PUBLIC_BRIDGE} up ${GATEWAY_IP}/24"
-    else
-        ${SSH} ${!CONTROLIP} "sudo ip link add link ${PUBLIC_BRIDGE} name ${PUBLIC_BRIDGE}.${GATEWAY_VLAN_ID} type vlan id ${GATEWAY_VLAN_ID}"
-        ${SSH} ${!CONTROLIP} "sudo ifconfig ${PUBLIC_BRIDGE} up"
-        ${SSH} ${!CONTROLIP} "sudo ifconfig ${PUBLIC_BRIDGE}.${GATEWAY_VLAN_ID} up ${GATEWAY_IP}/24"
-    fi
+    # Parameter values below are used in integration/test - changing them requires updates in intergration/test as well
+    EXTNET_GATEWAY_IP="10.10.10.250"
+    EXTNET_VLAN_ID=167
+    EXTNET_INTERNET_IP="10.9.9.9"
+    EXTNET_PNF_IP="10.10.10.253"
+    ${SSH} ${!CONTROLIP} "sudo ifconfig ${PUBLIC_BRIDGE} up ${EXTNET_GATEWAY_IP}/24"
+
+    # Control Node - external net PNF simulation
+    ${SSH} ${!CONTROLIP} "
+        sudo ip netns add pnf_ns;
+        sudo ip link add pnf_veth0 type veth peer name pnf_veth1;
+        sudo ip link set pnf_veth1 netns pnf_ns;
+        sudo ip link set pnf_veth0 up;
+        sudo ip netns exec pnf_ns ifconfig pnf_veth1 up ${EXTNET_PNF_IP}/24;
+        sudo ovs-vsctl add-port ${PUBLIC_BRIDGE} pnf_veth0;
+    "
+
+    # Control Node - external net internet address simulation
+    ${SSH} ${!CONTROLIP} "
+        sudo ip tuntap add dev internet_tap mode tap;
+        sudo ifconfig internet_tap up ${EXTNET_INTERNET_IP}/24;
+    "
 
     # Computes
     compute_index=1
@@ -766,7 +788,7 @@ echo "Starting Robot test suites ${SUITES} ..."
 # please add pybot -v arguments on a single line and alphabetized
 pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude \
     -v BUNDLEFOLDER:${BUNDLEFOLDER} \
-    -v BUNDLE_URL:${ACTUALBUNDLEURL} \
+    -v BUNDLE_URL:${ACTUAL_BUNDLE_URL} \
     -v CONTROLLER_USER:${USER} \
     -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack \
     -v HA_PROXY_IP:${HA_PROXY_IP} \
@@ -779,6 +801,7 @@ pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude \
     -v NUM_OPENSTACK_SITES:${NUM_OPENSTACK_SITES} \
     -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} \
     -v NUM_TOOLS_SYSTEM:${NUM_TOOLS_SYSTEM} \
+    -v ODL_SNAT_MODE:${ODL_SNAT_MODE} \
     -v ODL_STREAM:${DISTROSTREAM} \
     -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} \
     -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} \