jjb/integration/integration-apex-run-tests.sh

   1 #!/bin/bash -l
   2 # Activate robotframework virtualenv
   3 # ${ROBOT_VENV} comes from the integration-install-robotframework.sh
   4 # script.
   5 # shellcheck source=${ROBOT_VENV}/bin/activate disable=SC1091
   6 source ${ROBOT_VENV}/bin/activate
   7 source /tmp/common-functions.sh ${BUNDLEFOLDER}
   8 # Ensure we fail the job if any steps fail.
   9 set -ex -o pipefail
  10
  11 print_job_parameters
  12
  13 get_os_deploy
  14
  15 configure_karaf_log_for_apex $OPENSTACK_CONTROL_NODE_1_IP
  16
  17 # Swap out the ODL distribution
  18 DISTRO_UNDER_TEST=/tmp/odl.tar.gz
  19 wget --progress=dot:mega ${ACTUAL_BUNDLE_URL}
  20 UNZIPPED_DIR=`dirname "$(unzip -qql ${BUNDLE} | head -n1 | tr -s ' ' | cut -d' ' -f5-)"`
  21 unzip -q ${BUNDLE}
  22 tar czf ${DISTRO_UNDER_TEST} ${UNZIPPED_DIR}
  23 git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git /tmp/sdnvpn
  24 pushd /tmp/sdnvpn; git fetch https://gerrit.opnfv.org/gerrit/sdnvpn refs/changes/93/63293/1 && git checkout FETCH_HEAD; popd
  25 /tmp/sdnvpn/odl-pipeline/lib/odl_reinstaller.sh --pod-config ${WORKSPACE}/node.yaml --odl-artifact ${DISTRO_UNDER_TEST} --ssh-key-file ~/.ssh/robot_id_rsa
  26
  27 cat > /tmp/extra_node_configs.sh << EOF
  28 sudo jq -c '. + {"neutron::plugins::ovs::opendaylight::provider_mappings": ["datacentre:br-datacentre"]}' /etc/puppet/hieradata/config_step.json > tmp.$$.json && mv -f tmp.$$.json /etc/puppet/hieradata/config_step.json
  29 sudo puppet apply -e 'include tripleo::profile::base::neutron::plugins::ovs::opendaylight' -v
  30 sudo iptables -I INPUT -p udp -m multiport --dports 4789,9876,12345 -j ACCEPT
  31 sudo iptables -I INPUT -p tcp --dport 12345 -j ACCEPT
  32 sudo iptables --line-numbers -nvL
  33 EOF
  34 echo "cat extra_node_configs.sh"
  35 cat /tmp/extra_node_configs.sh
  36
  37 scp /tmp/extra_node_configs.sh  $OPENSTACK_CONTROL_NODE_1_IP:/tmp
  38 ssh $OPENSTACK_CONTROL_NODE_1_IP "sudo bash /tmp/extra_node_configs.sh"
  39 scp /tmp/extra_node_configs.sh  $OPENSTACK_COMPUTE_NODE_1_IP:/tmp
  40 ssh $OPENSTACK_COMPUTE_NODE_1_IP "sudo bash /tmp/extra_node_configs.sh"
  41 scp /tmp/extra_node_configs.sh  $OPENSTACK_COMPUTE_NODE_2_IP:/tmp
  42 ssh $OPENSTACK_COMPUTE_NODE_2_IP "sudo bash /tmp/extra_node_configs.sh"
  43
  44 USER=heat-admin
  45 openstack object save OPNFV-APEX-SNAPSHOTS overcloudrc
  46 source overcloudrc
  47 cat overcloudrc
  48 openstack hypervisor list
  49
  50 # Extra Configs needed for CSIT:
  51 wget -O ${WORKSPACE}/cirros-0.3.5-x86_64-disk.img http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img
  52 export ANSIBLE_HOST_KEY_CHECKING=False
  53 ansible-playbook -i ${OPENSTACK_CONTROL_NODE_1_IP}, -u heat-admin --key-file /tmp/id_rsa /tmp/opnfv_releng/jjb/cperf/cirros-upload.yaml.ansible -vvv
  54
  55 PHYSNET_WORK=datacentre
  56 BR_WORK=br-${PHYSNET_WORK}
  57 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "sudo ovs-vsctl --if-exists del-port br-int ${BR_WORK}"
  58 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "sudo ovs-vsctl --may-exist add-br ${BR_WORK} -- set bridge ${BR_WORK} other-config:disable-in-band=true other_config:hwaddr=f6:00:00:ff:01:01"
  59
  60 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "sudo ovs-vsctl --if-exists del-port br-int ${BR_WORK}"
  61 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "sudo ovs-vsctl --may-exist add-br ${BR_WORK} -- set bridge ${BR_WORK} other-config:disable-in-band=true other_config:hwaddr=f6:00:00:ff:01:02"
  62
  63 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "sudo ovs-vsctl --if-exists del-port br-int ${BR_WORK}"
  64 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "sudo ovs-vsctl --may-exist add-br ${BR_WORK} -- set bridge ${BR_WORK} other-config:disable-in-band=true other_config:hwaddr=f6:00:00:ff:01:03"
  65
  66 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "
  67         sudo ovs-vsctl add-port ${BR_WORK} compute_1_vxlan -- set interface compute_1_vxlan type=vxlan options:local_ip=${OPENSTACK_CONTROL_NODE_1_IP} options:remote_ip=${OPENSTACK_COMPUTE_NODE_1_IP} options:dst_port=9876 options:key=flow
  68 "
  69 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "
  70     sudo ovs-vsctl add-port ${BR_WORK} compute_2_vxlan -- set interface compute_2_vxlan type=vxlan options:local_ip=${OPENSTACK_CONTROL_NODE_1_IP} options:remote_ip=${OPENSTACK_COMPUTE_NODE_2_IP} options:dst_port=9876 options:key=flow
  71 "
  72 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "
  73         sudo ovs-vsctl add-port ${BR_WORK} control_1_vxlan -- set interface control_1_vxlan type=vxlan options:local_ip=${OPENSTACK_COMPUTE_NODE_1_IP} options:remote_ip=${OPENSTACK_CONTROL_NODE_1_IP} options:dst_port=9876 options:key=flow
  74 "
  75 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "
  76         sudo ovs-vsctl add-port ${BR_WORK} control_1_vxlan -- set interface control_1_vxlan type=vxlan options:local_ip=${OPENSTACK_COMPUTE_NODE_2_IP} options:remote_ip=${OPENSTACK_CONTROL_NODE_1_IP} options:dst_port=9876 options:key=flow
  77 "
  78
  79 # Control Node - PUBLIC_BRIDGE will act as the external router
  80 # Parameter values below are used in integration/test - changing them requires updates in intergration/test as well
  81 EXTNET_GATEWAY_IP="10.10.10.250"
  82 EXTNET_INTERNET_IP="10.9.9.9"
  83 EXTNET_PNF_IP="10.10.10.253"
  84 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "sudo ifconfig ${PUBLIC_BRIDGE} up ${EXTNET_GATEWAY_IP}/24"
  85
  86 # Control Node - external net PNF simulation
  87 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "
  88     sudo ip netns add pnf_ns;
  89     sudo ip link add pnf_veth0 type veth peer name pnf_veth1;
  90     sudo ip link set pnf_veth1 netns pnf_ns;
  91     sudo ip link set pnf_veth0 up;
  92     sudo ip netns exec pnf_ns ifconfig pnf_veth1 up ${EXTNET_PNF_IP}/24;
  93     sudo ovs-vsctl add-port ${PUBLIC_BRIDGE} pnf_veth0;
  94 "
  95 # Control Node - external net internet address simulation
  96 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "
  97     sudo ip tuntap add dev internet_tap mode tap;
  98     sudo ifconfig internet_tap up ${EXTNET_INTERNET_IP}/24;
  99 "
 100
 101 ssh ${OPENSTACK_CONTROL_NODE_1_IP} "sudo ovs-vsctl show"
 102 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "sudo ovs-vsctl show"
 103 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "sudo ovs-vsctl show"
 104
 105 get_test_suites SUITES
 106
 107 echo "Starting Robot test suites ${SUITES} ..."
 108 # please add robot -v arguments on a single line and alphabetized
 109 suite_num=0
 110 for suite in ${SUITES}; do
 111     # prepend an incremental counter to the suite name so that the full robot log combining all the suites as is done
 112     # in the rebot step below will list all the suites in chronological order as rebot seems to alphabetize them
 113     let "suite_num = suite_num + 1"
 114     suite_index="$(printf %02d ${suite_num})"
 115     suite_name="$(basename ${suite} | cut -d. -f1)"
 116     log_name="${suite_index}_${suite_name}"
 117     robot -N ${log_name} \
 118     -c critical -e exclude -e skip_if_${DISTROSTREAM} -e NON_GATE \
 119     --log log_${log_name}.html --report report_${log_name}.html --output output_${log_name}.xml \
 120     --removekeywords wuks \
 121     --removekeywords name:SetupUtils.Setup_Utils_For_Setup_And_Teardown \
 122     --removekeywords name:SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing \
 123     --removekeywords name:OpenStackOperations.Add_OVS_Logging_On_All_OpenStack_Nodes \
 124     -v BUNDLEFOLDER:${BUNDLEFOLDER} \
 125     -v BUNDLE_URL:${ACTUAL_BUNDLE_URL} \
 126     -v CMP_INSTANCES_SHARED_PATH:/var/instances \
 127     -v CONTROLLERFEATURES:"${CONTROLLERFEATURES}" \
 128     -v CONTROLLER_USER:${USER} \
 129     -v DEFAULT_LINUX_PROMPT:\$ \
 130     -v DEFAULT_LINUX_PROMPT_STRICT:]\$ \
 131     -v DEFAULT_USER:${USER} \
 132     -v ENABLE_ITM_DIRECT_TUNNELS:${ENABLE_ITM_DIRECT_TUNNELS} \
 133     -v HA_PROXY_IP:$ODL_SYSTEM_IP \
 134     -v JDKVERSION:${JDKVERSION} \
 135     -v JENKINS_WORKSPACE:${WORKSPACE} \
 136     -v KARAF_LOG:/opt/opendaylight/data/log/karaf.log \
 137     -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} \
 138     -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} \
 139     -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} \
 140     -v NUM_TOOLS_SYSTEM:${NUM_TOOLS_SYSTEM} \
 141     -v ODL_RESTCONF_PASSWORD:$SDN_CONTROLLER_PASSWORD \
 142     -v ODL_SNAT_MODE:${ODL_SNAT_MODE} \
 143     -v ODL_STREAM:${DISTROSTREAM} \
 144     -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} \
 145     -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} \
 146     -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_1_IP} \
 147     -v OS_CONTROL_NODE_1_IP:${OPENSTACK_CONTROL_NODE_1_IP} \
 148     -v OPENSTACK_BRANCH:${OPENSTACK_BRANCH} \
 149     -v OS_COMPUTE_1_IP:${OPENSTACK_COMPUTE_NODE_1_IP} \
 150     -v OS_COMPUTE_2_IP:${OPENSTACK_COMPUTE_NODE_2_IP} \
 151     -v OPENSTACK_TOPO:${OPENSTACK_TOPO} \
 152     -v OS_USER:${USER} \
 153     -v PUBLIC_PHYSICAL_NETWORK:${PHYSNET_WORK} \
 154     -v RESTCONFPORT:8081 \
 155     -v SECURITY_GROUP_MODE:${SECURITY_GROUP_MODE} \
 156     -v SSH_KEY:robot_id_rsa \
 157     -v TOOLS_SYSTEM_IP: \
 158     -v USER_HOME:${HOME} \
 159     -v WORKSPACE:/tmp \
 160     ${TESTOPTIONS} ${suite} || true
 161 done
 162 #rebot exit codes seem to be different
 163 rebot --output ${WORKSPACE}/output.xml --log log_full.html --report report.html -N openstack output_*.xml || true
 164
 165 echo "Collecting logs"
 166 mkdir -p ${WORKSPACE}/archives
 167
 168 ssh ${ODL_SYSTEM_IP} "mkdir /tmp/controller_sos; sudo sosreport --all-logs --batch --build --tmp-dir /tmp/controller_sos/ --name controller_report -o networking,openstack_glance,openstack_neutron,openstack_nova,opendaylight,openvswitch"
 169 ssh ${ODL_SYSTEM_IP} "sudo chmod -R 0777 /tmp/controller_sos/; tar cvzf /tmp/controller_sos.tar.gz /tmp/controller_sos"
 170 scp ${ODL_SYSTEM_IP}:/tmp/controller_sos.tar.gz ${WORKSPACE}/archives/controller_sos.tar.gz
 171 gunzip ${WORKSPACE}/archives/controller_sos.tar.gz
 172 tar -xvf ${WORKSPACE}/archives/controller_sos.tar -C ${WORKSPACE}/archives/
 173
 174 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "mkdir /tmp/compute_1_sos; sudo sosreport --all-logs --batch --build --tmp-dir /tmp/compute_1_sos/ --name compute_1_report -o networking,openstack_glance,openstack_neutron,openstack_nova,opendaylight,openvswitch"
 175 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "sudo chmod -R 0777 /tmp/compute_1_sos/; tar cvzf /tmp/compute_1_sos.tar.gz /tmp/compute_1_sos"
 176 scp ${OPENSTACK_COMPUTE_NODE_1_IP}:/tmp/compute_1_sos.tar.gz ${WORKSPACE}/archives/compute_1_sos.tar.gz
 177 gunzip ${WORKSPACE}/archives/compute_1_sos.tar.gz
 178 tar -xvf ${WORKSPACE}/archives/compute_1_sos.tar -C ${WORKSPACE}/archives/
 179
 180 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "mkdir /tmp/compute_2_sos; sudo sosreport --all-logs --batch --build --tmp-dir /tmp/compute_2_sos/ --name compute_2_report -o networking,openstack_glance,openstack_neutron,openstack_nova,opendaylight,openvswitch"
 181 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "sudo chmod -R 0777 /tmp/compute_2_sos/; tar cvzf /tmp/compute_2_sos.tar.gz /tmp/compute_2_sos"
 182 scp ${OPENSTACK_COMPUTE_NODE_2_IP}:/tmp/compute_2_sos.tar.gz ${WORKSPACE}/archives/compute_2_sos.tar.gz
 183 gunzip ${WORKSPACE}/archives/compute_2_sos.tar.gz
 184 tar -xvf ${WORKSPACE}/archives/compute_2_sos.tar -C ${WORKSPACE}/archives/
 185
 186 mv ${WORKSPACE}/archives/tmp/* ${WORKSPACE}/archives/
 187 rm -rf ${WORKSPACE}/archives/tmp
 188
 189 true  # perhaps Jenkins is testing last exit code
 190 # vim: ts=4 sw=4 sts=4 et ft=sh :