From 7d2adf24cca1a990928270e0a1aba14bc8c414df Mon Sep 17 00:00:00 2001 From: Sandra Biton Date: Thu, 5 Jan 2017 15:56:54 +0200 Subject: [PATCH] Add support for multi-site Add support for multi-site netvirt for federation CSIT Depends-On: https://git.opendaylight.org/gerrit/#/c/49146/ Change-Id: I9f7b66177eac7d237f8509828a45e3a012213d2a Signed-off-by: Sandra Biton Signed-off-by: Guy Belotzerkovsky Signed-off-by: Alon Kochba --- ...-integration-deploy-controller-run-test.sh | 40 +- ...w-integration-deploy-openstack-run-test.sh | 481 ++++++++++-------- ...ude-raw-integration-get-slave-addresses.sh | 32 +- jjb/integration/integration-templates.yaml | 5 + jjb/netvirt/netvirt-csit-multi-openstack.yaml | 48 ++ jjb/releng-defaults.yaml | 1 + 6 files changed, 364 insertions(+), 243 deletions(-) diff --git a/jjb/integration/include-raw-integration-deploy-controller-run-test.sh b/jjb/integration/include-raw-integration-deploy-controller-run-test.sh index f168aaefc..5dd16bb0d 100644 --- a/jjb/integration/include-raw-integration-deploy-controller-run-test.sh +++ b/jjb/integration/include-raw-integration-deploy-controller-run-test.sh @@ -11,13 +11,6 @@ if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then AKKACONF=/tmp/${BUNDLEFOLDER}/configuration/initial/akka.conf MODULESCONF=/tmp/${BUNDLEFOLDER}/configuration/initial/modules.conf MODULESHARDSCONF=/tmp/${BUNDLEFOLDER}/configuration/initial/module-shards.conf - # Create the string for odl nodes - odl_node_list="${ODL_SYSTEM_1_IP}" - for i in `seq 2 ${NUM_ODL_SYSTEM}` ; do - CONTROLLERIP=ODL_SYSTEM_${i}_IP - odl_node_list="${odl_node_list} ${!CONTROLLERIP}" - done - echo ${odl_node_list} fi if [ ${CONTROLLERSCOPE} == 'all' ]; then @@ -107,7 +100,7 @@ if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then fi echo "Configuring cluster" - /tmp/${BUNDLEFOLDER}/bin/configure_cluster.sh \$1 ${odl_node_list} + /tmp/${BUNDLEFOLDER}/bin/configure_cluster.sh \$1 \$2 echo "Dump akka.conf" cat ${AKKACONF} @@ -182,13 +175,26 @@ exit_on_log_file_message 'server is unhealthy' EOF -# Copy over the config script to controller and execute it. -for i in `seq 1 ${NUM_ODL_SYSTEM}` +[ "$NUM_OPENSTACK_SITES" ] || NUM_OPENSTACK_SITES=1 +NUM_ODLS_PER_SITE=$((NUM_ODL_SYSTEM / NUM_OPENSTACK_SITES)) +for i in `seq 1 ${NUM_OPENSTACK_SITES}` do - CONTROLLERIP=ODL_SYSTEM_${i}_IP - echo "Execute the configuration script on controller ${!CONTROLLERIP}" - scp ${WORKSPACE}/configuration-script.sh ${!CONTROLLERIP}:/tmp - ssh ${!CONTROLLERIP} "bash /tmp/configuration-script.sh ${i}" + # Get full list of ODL nodes for this site + odl_node_list= + for j in `seq 1 ${NUM_ODLS_PER_SITE}` + do + odl_ip=ODL_SYSTEM_$(((i - 1) * NUM_ODLS_PER_SITE + j))_IP + odl_node_list="${odl_node_list} ${!odl_ip}" + done + + for j in `seq 1 ${NUM_ODLS_PER_SITE}` + do + odl_ip=ODL_SYSTEM_$(((i - 1) * NUM_ODLS_PER_SITE + j))_IP + # Copy over the config script to controller and execute it (parameters are used only for cluster) + echo "Execute the configuration script on controller ${!odl_ip} for index $j with node list ${odl_node_list}" + scp ${WORKSPACE}/configuration-script.sh ${!odl_ip}:/tmp + ssh ${!odl_ip} "bash /tmp/configuration-script.sh ${j} '${odl_node_list}'" + done done echo "Locating config plan to use..." @@ -217,12 +223,16 @@ do ssh ${!CONTROLLERIP} "bash /tmp/startup-script.sh" done +seed_index=1 for i in `seq 1 ${NUM_ODL_SYSTEM}` do CONTROLLERIP=ODL_SYSTEM_${i}_IP echo "Execute the post startup script on controller ${!CONTROLLERIP}" scp ${WORKSPACE}/post-startup-script.sh ${!CONTROLLERIP}:/tmp - ssh ${!CONTROLLERIP} "bash /tmp/post-startup-script.sh ${i}" + ssh ${!CONTROLLERIP} "bash /tmp/post-startup-script.sh $(( seed_index++ ))" + if [ $(( $i % (${NUM_ODL_SYSTEM} / ${NUM_OPENSTACK_SITES}) )) == 0 ]; then + seed_index=1 + fi done echo "Cool down for ${COOLDOWN_PERIOD} seconds :)..." diff --git a/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh b/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh index 9642c3469..c79fde989 100644 --- a/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh +++ b/jjb/integration/include-raw-integration-deploy-openstack-run-test.sh @@ -19,13 +19,16 @@ echo "#################################################" SSH="ssh -t -t" function create_control_node_local_conf { -local_conf_file_name=${WORKSPACE}/local.conf_control +HOSTIP=$1 +MGRIP=$2 +ODL_OVS_MANAGERS="$3" #Needs to be removed if [ "${ODL_ML2_BRANCH}" != "stable/ocata" ]; then RECLONE=no else RECLONE=yes fi +local_conf_file_name=${WORKSPACE}/local.conf_control_${HOSTIP} cat > ${local_conf_file_name} << EOF [[local|localrc]] LOGFILE=stack.sh.log @@ -70,7 +73,7 @@ if [ "${OPENSTACK_BRANCH}" == "master" ] || [ "${OPENSTACK_BRANCH}" == "stable/o echo "enable_service placement-api" >> ${local_conf_file_name} fi cat >> ${local_conf_file_name} << EOF -HOST_IP=$OPENSTACK_CONTROL_NODE_IP +HOST_IP=${HOSTIP} SERVICE_HOST=\$HOST_IP NEUTRON_CREATE_INITIAL_NETWORKS=${CREATE_INITIAL_NETWORKS} @@ -97,6 +100,7 @@ ODL_MODE=externalodl ODL_PORT_BINDING_CONTROLLER=${ODL_ML2_PORT_BINDING} LIBVIRT_TYPE=qemu +ODL_MGR_IP=${MGRIP} NEUTRON_LBAAS_SERVICE_PROVIDERV2=${LBAAS_SERVICE_PROVIDER} # Only relevant if neutron-lbaas plugin is enabled EOF @@ -115,37 +119,15 @@ if [ "${ODL_ML2_DRIVER_VERSION}" == "v2" ]; then echo "ODL_V2DRIVER=True" >> ${local_conf_file_name} fi -if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then -odl_list=${ODL_SYSTEM_1_IP} -for i in `seq 2 ${NUM_ODL_SYSTEM}` -do -odlip=ODL_SYSTEM_${i}_IP -odl_list=${odl_list},${!odlip} -done -if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then -HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM} -odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP -odl_mgr_ip=${!odlmgrip} -else -odl_mgr_ip=${ODL_SYSTEM_1_IP} -fi -cat >> ${local_conf_file_name} << EOF -ODL_OVS_MANAGERS=${odl_list} -ODL_MGR_IP=${odl_mgr_ip} -EOF -else -cat >> ${local_conf_file_name} << EOF -ODL_MGR_IP=${ODL_SYSTEM_1_IP} -EOF -fi +echo "ODL_OVS_MANAGERS=${ODL_OVS_MANAGERS}" >> ${local_conf_file_name} # if we are using the old netvirt impl, as determined by the feature name # odl-ovsdb-openstack (note: new impl is odl-netvirt-openstack) then we # want ODL_L3 to be True. New impl wants it False if [[ ${CONTROLLERFEATURES} == *"odl-ovsdb-openstack"* ]]; then - ODL_L3=True + ODL_L3=True else - ODL_L3=False + ODL_L3=False fi # if we are using the new netvirt impl, as determined by the feature name @@ -212,6 +194,9 @@ cat ${local_conf_file_name} function create_compute_node_local_conf { HOSTIP=$1 +SERVICEHOST=$2 +MGRIP=$3 +ODL_OVS_MANAGERS="$4" #Needs to be removed if [ "${ODL_ML2_BRANCH}" != "stable/ocata" ]; then RECLONE=no @@ -237,7 +222,7 @@ NOVA_VNC_ENABLED=True MULTI_HOST=1 ENABLED_SERVICES=${ENABLED_SERVICES} HOST_IP=${HOSTIP} -SERVICE_HOST=${OPENSTACK_CONTROL_NODE_IP} +SERVICE_HOST=${SERVICEHOST} Q_PLUGIN=ml2 ENABLE_TENANT_TUNNELS=True @@ -259,6 +244,7 @@ ADMIN_PASSWORD=admin ODL_MODE=compute ODL_PORT_BINDING_CONTROLLER=${ODL_ML2_PORT_BINDING} LIBVIRT_TYPE=qemu +ODL_MGR_IP=${MGRIP} EOF if [[ "${ENABLE_OS_PLUGINS}" =~ networking-odl ]]; then @@ -267,29 +253,7 @@ enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH} EOF fi -if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then -odl_list=${ODL_SYSTEM_1_IP} -for i in `seq 2 ${NUM_ODL_SYSTEM}` -do -odlip=ODL_SYSTEM_${i}_IP -odl_list=${odl_list},${!odlip} -done -if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then -HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM} -odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP -odl_mgr_ip=${!odlmgrip} -else -odl_mgr_ip=${ODL_SYSTEM_1_IP} -fi -cat >> ${local_conf_file_name} << EOF -ODL_OVS_MANAGERS=${odl_list} -ODL_MGR_IP=${odl_mgr_ip} -EOF -else -cat >> ${local_conf_file_name} << EOF -ODL_MGR_IP=${ODL_SYSTEM_1_IP} -EOF -fi +echo "ODL_OVS_MANAGERS=${ODL_OVS_MANAGERS}" >> ${local_conf_file_name} # if we are using the new netvirt impl, as determined by the feature name # odl-netvirt-openstack (note: old impl is odl-ovsdb-openstack) then we @@ -317,9 +281,8 @@ cat ${local_conf_file_name} } function configure_haproxy_for_neutron_requests () { -HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM} -odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP -ha_proxy_ip=${!odlmgrip} +MGRIP=$1 +ODL_IPS=(${2//,/ }) cat > ${WORKSPACE}/install_ha_proxy.sh<< EOF sudo systemctl stop firewalld @@ -348,32 +311,37 @@ defaults timeout check 10s listen opendaylight - bind ${ha_proxy_ip}:8080 + bind ${MGRIP}:8080 balance source EOF -for i in `seq 1 ${NUM_ODL_SYSTEM}` +odlindex=1 +for odlip in ${ODL_IPS[*]} do -odlip=ODL_SYSTEM_${i}_IP cat >> ${WORKSPACE}/haproxy.cfg << EOF - server controller-$i ${!odlip}:8080 check fall 5 inter 2000 rise 2 + server controller-${odlindex} ${odlip}:8080 check fall 5 inter 2000 rise 2 EOF +odlindex=$((odlindex+1)) done cat >> ${WORKSPACE}/haproxy.cfg << EOF listen opendaylight_rest - bind ${ha_proxy_ip}:8181 + bind ${MGRIP}:8181 balance source EOF -for i in `seq 1 ${NUM_ODL_SYSTEM}` +odlindex=1 +for odlip in ${ODL_IPS[*]} do -odlip=ODL_SYSTEM_${i}_IP cat >> ${WORKSPACE}/haproxy.cfg << EOF - server controller-rest-$i ${!odlip}:8181 check fall 5 inter 2000 rise 2 + server controller-rest-${odlindex} ${odlip}:8181 check fall 5 inter 2000 rise 2 EOF +odlindex=$((odlindex+1)) done +echo "Dump haproxy.cfg" +cat ${WORKSPACE}/haproxy.cfg + cat > ${WORKSPACE}/deploy_ha_proxy.sh<< EOF sudo chown haproxy:haproxy /tmp/haproxy.cfg sudo sed -i 's/\\/etc\\/haproxy\\/haproxy.cfg/\\/tmp\\/haproxy.cfg/g' /usr/lib/systemd/system/haproxy.service @@ -384,14 +352,14 @@ sudo netstat -tunpl sudo systemctl status haproxy true EOF -scp ${WORKSPACE}/install_ha_proxy.sh ${ha_proxy_ip}:/tmp -${SSH} ${ha_proxy_ip} "sudo bash /tmp/install_ha_proxy.sh" -scp ${WORKSPACE}/haproxy.cfg ${ha_proxy_ip}:/tmp -scp ${WORKSPACE}/deploy_ha_proxy.sh ${ha_proxy_ip}:/tmp -${SSH} ${ha_proxy_ip} "sudo bash /tmp/deploy_ha_proxy.sh" +scp ${WORKSPACE}/install_ha_proxy.sh ${MGRIP}:/tmp +${SSH} ${MGRIP} "sudo bash /tmp/install_ha_proxy.sh" +scp ${WORKSPACE}/haproxy.cfg ${MGRIP}:/tmp +scp ${WORKSPACE}/deploy_ha_proxy.sh ${MGRIP}:/tmp +${SSH} ${MGRIP} "sudo bash /tmp/deploy_ha_proxy.sh" } -function collect_logs_and_exit (){ +function collect_logs_and_exit () { set +e # We do not want to create red dot just because something went wrong while fetching logs. for i in `seq 1 ${NUM_ODL_SYSTEM}` do @@ -428,21 +396,25 @@ done mkdir -p ${WORKSPACE}/archives # Control Node -OS_CTRL_FOLDER="control" -mkdir -p ${OS_CTRL_FOLDER} -scp ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/nohup.out ${OS_CTRL_FOLDER}/stack.log -scp ${OPENSTACK_CONTROL_NODE_IP}:/var/log/openvswitch/ovs-vswitchd.log ${OS_CTRL_FOLDER}/ovs-vswitchd.log -scp ${OPENSTACK_CONTROL_NODE_IP}:/etc/neutron/neutron.conf ${OS_CTRL_FOLDER}/neutron.conf -scp ${OPENSTACK_CONTROL_NODE_IP}:/etc/nova/nova.conf ${OS_CTRL_FOLDER}/nova.conf -rsync -avhe ssh ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/logs/* ${OS_CTRL_FOLDER} # rsync to prevent copying of symbolic links -scp extra_debug.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log" -scp ${OPENSTACK_CONTROL_NODE_IP}:/tmp/extra_debug.log ${OS_CTRL_FOLDER}/extra_debug.log -mv local.conf_control ${OS_CTRL_FOLDER}/local.conf -mv ${OS_CTRL_FOLDER} ${WORKSPACE}/archives/ +for i in `seq 1 ${NUM_OPENSTACK_CONTROL_NODES}` +do + OS_CTRL_IP=OPENSTACK_CONTROL_NODE_${i}_IP + OS_CTRL_FOLDER="control_${i}" + mkdir -p ${OS_CTRL_FOLDER} + scp ${!OS_CTRL_IP}:/opt/stack/devstack/nohup.out ${OS_CTRL_FOLDER}/stack.log + scp ${!OS_CTRL_IP}:/var/log/openvswitch/ovs-vswitchd.log ${OS_CTRL_FOLDER}/ovs-vswitchd.log + scp ${!OS_CTRL_IP}:/etc/neutron/neutron.conf ${OS_CTRL_FOLDER}/neutron.conf + scp ${!OS_CTRL_IP}:/etc/nova/nova.conf ${OS_CTRL_FOLDER}/nova.conf + rsync -avhe ssh ${!OS_CTRL_IP}:/opt/stack/logs/* ${OS_CTRL_FOLDER} # rsync to prevent copying of symbolic links + scp extra_debug.sh ${!OS_CTRL_IP}:/tmp + ${SSH} ${!OS_CTRL_IP} "bash /tmp/extra_debug.sh > /tmp/extra_debug.log" + scp ${!OS_CTRL_IP}:/tmp/extra_debug.log ${OS_CTRL_FOLDER}/extra_debug.log + mv local.conf_control ${OS_CTRL_FOLDER}/local.conf + mv ${OS_CTRL_FOLDER} ${WORKSPACE}/archives/ +done # Compute Nodes -for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))` +for i in `seq 1 ${NUM_OPENSTACK_COMPUTE_NODES}` do OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP OS_COMPUTE_FOLDER="compute_${i}" @@ -462,13 +434,13 @@ ls local.conf* | xargs -I % mv % %.log # Tempest DEVSTACK_TEMPEST_DIR="/opt/stack/tempest" -if $(ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '[ -f ${DEVSTACK_TEMPEST_DIR}/.testrepository/0 ]'"); then # if Tempest results exist - ssh ${OPENSTACK_CONTROL_NODE_IP} "for I in \$(sudo ls ${DEVSTACK_TEMPEST_DIR}/.testrepository/ | grep -E '^[0-9]+$'); do sudo sh -c \"${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/subunit-1to2 < ${DEVSTACK_TEMPEST_DIR}/.testrepository/\${I} >> ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt\"; done" - ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/python ${DEVSTACK_TEMPEST_DIR}/.tox/tempest/lib/python2.7/site-packages/os_testr/subunit2html.py ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt ${DEVSTACK_TEMPEST_DIR}/tempest_results.html'" +if $(ssh ${OPENSTACK_CONTROL_NODE_1_IP} "sudo sh -c '[ -f ${DEVSTACK_TEMPEST_DIR}/.testrepository/0 ]'"); then # if Tempest results exist + ssh ${OPENSTACK_CONTROL_NODE_1_IP} "for I in \$(sudo ls ${DEVSTACK_TEMPEST_DIR}/.testrepository/ | grep -E '^[0-9]+$'); do sudo sh -c \"${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/subunit-1to2 < ${DEVSTACK_TEMPEST_DIR}/.testrepository/\${I} >> ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt\"; done" + ssh ${OPENSTACK_CONTROL_NODE_1_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/python ${DEVSTACK_TEMPEST_DIR}/.tox/tempest/lib/python2.7/site-packages/os_testr/subunit2html.py ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt ${DEVSTACK_TEMPEST_DIR}/tempest_results.html'" TEMPEST_LOGS_DIR=${WORKSPACE}/archives/tempest mkdir -p ${TEMPEST_LOGS_DIR} - scp ${OPENSTACK_CONTROL_NODE_IP}:${DEVSTACK_TEMPEST_DIR}/tempest_results.html ${TEMPEST_LOGS_DIR} - scp ${OPENSTACK_CONTROL_NODE_IP}:${DEVSTACK_TEMPEST_DIR}/tempest.log ${TEMPEST_LOGS_DIR} + scp ${OPENSTACK_CONTROL_NODE_1_IP}:${DEVSTACK_TEMPEST_DIR}/tempest_results.html ${TEMPEST_LOGS_DIR} + scp ${OPENSTACK_CONTROL_NODE_1_IP}:${DEVSTACK_TEMPEST_DIR}/tempest.log ${TEMPEST_LOGS_DIR} mv ${WORKSPACE}/tempest_output* ${TEMPEST_LOGS_DIR} fi } @@ -501,18 +473,12 @@ cd devstack git checkout $OPENSTACK_BRANCH EOF -echo "Create HAProxy if needed" -if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then - echo "Need to configure HAProxy" - configure_haproxy_for_neutron_requests -fi - +[ "$NUM_OPENSTACK_SITES" ] || NUM_OPENSTACK_SITES=1 +compute_index=1 +odl_index=1 os_node_list=() -echo "Stack the Control Node" -scp ${WORKSPACE}/get_devstack.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/get_devstack.sh" -create_control_node_local_conf -scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf +os_interval=$(( ${NUM_OPENSTACK_SYSTEM} / ${NUM_OPENSTACK_SITES} )) +ha_proxy_index=${os_interval} cat > "${WORKSPACE}/manual_install_package.sh" << EOF cd /opt/stack @@ -520,52 +486,83 @@ git clone "\$1" cd "\$2" git checkout "\$3" sudo python setup.py install - EOF +for i in `seq 1 ${NUM_OPENSTACK_SITES}` +do + if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then + echo "Configure HAProxy" + ODL_HAPROXYIP_PARAM=OPENSTACK_HAPROXY_${i}_IP + ha_proxy_index=$(( $ha_proxy_index + $os_interval )) + odl_index=$(((i - 1) * 3 + 1)) + ODL_IP_PARAM1=ODL_SYSTEM_$((odl_index++))_IP + ODL_IP_PARAM2=ODL_SYSTEM_$((odl_index++))_IP + ODL_IP_PARAM3=ODL_SYSTEM_$((odl_index++))_IP + ODLMGRIP[$i]=${!ODL_HAPROXYIP_PARAM} # ODL Northbound uses HAProxy VIP + ODL_OVS_MGRS[$i]="${!ODL_IP_PARAM1},${!ODL_IP_PARAM2},${!ODL_IP_PARAM3}" # OVSDB connects to all ODL IPs + + configure_haproxy_for_neutron_requests ${!ODL_HAPROXYIP_PARAM} "${ODL_OVS_MGRS[$i]}" + else + ODL_IP_PARAM=ODL_SYSTEM_${i}_IP + ODL_OVS_MGRS[$i]="${!ODL_IP_PARAM}" # ODL Northbound uses ODL IP + ODLMGRIP[$i]=${!ODL_IP_PARAM} # OVSDB connects to ODL IP + fi +done -# Workworund for successful stacking with Mitaka -if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then - - # Workaround for problems with latest versions/specified versions in requirements of openstack - # Openstacksdk,libvirt-python -> the current version does not work with Mitaka diue to some requirements - # conflict and breaks when trying to stack - # paramiko -> Problems with tempest tests due to paramiko incompatibility with pycrypto. - # the problem has been solved with version 1.17. If the latest version of paramiko is used, it causes - # other timeout problems - ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt; sed -i /paramiko/d upper-constraints.txt" - scp "${WORKSPACE}/manual_install_package.sh" "${OPENSTACK_CONTROL_NODE_IP}:/tmp" - ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install deprecation" - # Fix for recent requirements update in the master branch of the sdk.The section must be replaced with a better fix. - ssh "${OPENSTACK_CONTROL_NODE_IP}" "sh /tmp/manual_install_package.sh https://github.com/openstack/python-openstacksdk python-openstacksdk 0.9.14" - ssh "${OPENSTACK_CONTROL_NODE_IP}" "sh /tmp/manual_install_package.sh https://github.com/paramiko/paramiko paramiko 1.17" -fi - -ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &" -ssh ${OPENSTACK_CONTROL_NODE_IP} "ps -ef | grep stack.sh" -ssh ${OPENSTACK_CONTROL_NODE_IP} "ls -lrt /opt/stack/devstack/nohup.out" -os_node_list+=(${OPENSTACK_CONTROL_NODE_IP}) - -#Workaround for stable/newton jobs -if [ "${ODL_ML2_BRANCH}" == "stable/newton" ]; then - ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/newton; sed -i /appdirs/d upper-constraints.txt" -fi +for i in `seq 1 ${NUM_OPENSTACK_CONTROL_NODES}` +do + echo "Stack the Control Node" + CONTROLIP=OPENSTACK_CONTROL_NODE_${i}_IP + scp ${WORKSPACE}/get_devstack.sh ${!CONTROLIP}:/tmp + ${SSH} ${!CONTROLIP} "bash /tmp/get_devstack.sh" + create_control_node_local_conf ${!CONTROLIP} ${ODLMGRIP[$i]} "${ODL_OVS_MGRS[$i]}" + scp ${WORKSPACE}/local.conf_control_${!CONTROLIP} ${!CONTROLIP}:/opt/stack/devstack/local.conf + if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then + # Workaround for problems with latest versions/specified versions in requirements of openstack + # Openstacksdk,libvirt-python -> the current version does not work with Mitaka diue to some requirements + # conflict and breaks when trying to stack + # paramiko -> Problems with tempest tests due to paramiko incompatibility with pycrypto. + # the problem has been solved with version 1.17. If the latest version of paramiko is used, it causes + # other timeout problems + ssh ${!CONTROLIP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt; sed -i /paramiko/d upper-constraints.txt" + scp "${WORKSPACE}/manual_install_package.sh" "${!CONTROLIP}:/tmp" + ssh ${!CONTROLIP} "sudo pip install deprecation" + # Fix for recent requirements update in the master branch of the sdk.The section must be replaced with a better fix. + ssh "${!CONTROLIP}" "sh /tmp/manual_install_package.sh https://github.com/openstack/python-openstacksdk python-openstacksdk 0.9.14" + ssh "${!CONTROLIP}" "sh /tmp/manual_install_package.sh https://github.com/paramiko/paramiko paramiko 1.17" + fi + ssh ${!CONTROLIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &" + ssh ${!CONTROLIP} "ps -ef | grep stack.sh" + ssh ${!CONTROLIP} "ls -lrt /opt/stack/devstack/nohup.out" + os_node_list+=(${!CONTROLIP}) + + #Workaround for stable/newton jobs + if [ "${ODL_ML2_BRANCH}" == "stable/newton" ]; then + ssh ${!CONTROLIP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/newton; sed -i /appdirs/d upper-constraints.txt" + fi +done -for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))` +for i in `seq 1 ${NUM_OPENSTACK_COMPUTE_NODES}` do + echo "Stack the Compute Node" + NUM_COMPUTES_PER_SITE=$((NUM_OPENSTACK_COMPUTE_NODES / NUM_OPENSTACK_SITES)) + SITE_INDEX=$((((i - 1) / NUM_COMPUTES_PER_SITE) + 1)) # We need the site index to infer the control node IP for this compute COMPUTEIP=OPENSTACK_COMPUTE_NODE_${i}_IP + CONTROLIP=OPENSTACK_CONTROL_NODE_${SITE_INDEX}_IP scp ${WORKSPACE}/get_devstack.sh ${!COMPUTEIP}:/tmp ${SSH} ${!COMPUTEIP} "bash /tmp/get_devstack.sh" - create_compute_node_local_conf ${!COMPUTEIP} + create_compute_node_local_conf ${!COMPUTEIP} ${!CONTROLIP} ${ODLMGRIP[$SITE_INDEX]} "${ODL_OVS_MGRS[$SITE_INDEX]}" scp ${WORKSPACE}/local.conf_compute_${!COMPUTEIP} ${!COMPUTEIP}:/opt/stack/devstack/local.conf if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then - ssh ${!COMPUTEIP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /libvirt-python/d upper-constraints.txt" + ssh ${!COMPUTEIP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /libvirt-python/d upper-constraints.txt" fi ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &" ssh ${!COMPUTEIP} "ps -ef | grep stack.sh" os_node_list+=(${!COMPUTEIP}) done +echo ${os_node_list[*]} + cat > ${WORKSPACE}/check_stacking.sh << EOF > /tmp/stack_progress ps -ef | grep "stack.sh" | grep -v grep @@ -616,116 +613,136 @@ done fi done -#Need to disable firewalld and iptables in control node -echo "Stop Firewall in Control Node for compute nodes to be able to reach the ports and add to hypervisor-list" -scp ${WORKSPACE}/disable_firewall.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo bash /tmp/disable_firewall.sh" -echo "sleep for a minute and print hypervisor-list" -sleep 60 -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; source openrc admin admin; nova hypervisor-list" -# in the case that we are doing openstack (control + compute) all in one node, then the number of hypervisors -# will be the same as the number of openstack systems. However, if we are doing multinode openstack then the -# assumption is we have a single control node and the rest are compute nodes, so the number of expected hypervisors -# is one less than the total number of openstack systems -if [ "${NUM_OPENSTACK_SYSTEM}" -eq 1 ]; then - expected_num_hypervisors=1 -else - expected_num_hypervisors=$((NUM_OPENSTACK_SYSTEM - 1)) -fi -num_hypervisors=$(${SSH} ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; source openrc admin admin; openstack hypervisor list -f value | wc -l" | tail -1 | tr -d "\r") -if ! [ "${num_hypervisors}" ] || ! [ ${num_hypervisors} -eq ${expected_num_hypervisors} ]; then - echo "Error: Only $num_hypervisors hypervisors detected, expected $expected_num_hypervisors" - collect_logs_and_exit - exit 1 -fi - -#Need to disable firewalld and iptables in compute nodes as well -for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))` +NUM_COMPUTES_PER_SITE=$((NUM_OPENSTACK_COMPUTE_NODES / NUM_OPENSTACK_SITES)) +for i in `seq 1 ${NUM_OPENSTACK_SITES}` do - OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP - scp ${WORKSPACE}/disable_firewall.sh "${!OSIP}:/tmp" - ${SSH} "${!OSIP}" "sudo bash /tmp/disable_firewall.sh" -done - -# upgrading pip, urllib3 and httplib2 so that tempest tests can be run on ${OPENSTACK_CONTROL_NODE_IP} -# this needs to happen after devstack runs because it seems devstack is pulling in specific versions -# of these libs that are not working for tempest. -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install --upgrade pip" -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install urllib3 --upgrade" -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install httplib2 --upgrade" + echo "Configure the Control Node" + CONTROLIP=OPENSTACK_CONTROL_NODE_${i}_IP + + #Need to disable firewalld and iptables in control node + echo "Stop Firewall in Control Node for compute nodes to be able to reach the ports and add to hypervisor-list" + scp ${WORKSPACE}/disable_firewall.sh ${!CONTROLIP}:/tmp + ${SSH} ${!CONTROLIP} "sudo bash /tmp/disable_firewall.sh" + + echo "sleep for 60s and print hypervisor-list" + sleep 60 + ${SSH} ${!CONTROLIP} "cd /opt/stack/devstack; source openrc admin admin; nova hypervisor-list" + # in the case that we are doing openstack (control + compute) all in one node, then the number of hypervisors + # will be the same as the number of openstack systems. However, if we are doing multinode openstack then the + # assumption is we have a single control node and the rest are compute nodes, so the number of expected hypervisors + # is one less than the total number of openstack systems + if [ $((NUM_OPENSTACK_SYSTEM / NUM_OPENSTACK_SITES - 1)) -eq 1 ]; then + expected_num_hypervisors=1 + else + expected_num_hypervisors=${NUM_COMPUTES_PER_SITE} + fi + num_hypervisors=$(${SSH} ${!CONTROLIP} "cd /opt/stack/devstack; source openrc admin admin; openstack hypervisor list -f value | wc -l" | tail -1 | tr -d "\r") + if ! [ "${num_hypervisors}" ] || ! [ ${num_hypervisors} -eq ${expected_num_hypervisors} ]; then + echo "Error: Only $num_hypervisors hypervisors detected, expected $expected_num_hypervisors" + collect_logs_and_exit + exit 1 + fi -for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))` -do - IP_VAR=OPENSTACK_COMPUTE_NODE_${i}_IP - COMPUTE_IPS[$((i-1))]=${!IP_VAR} -done + # upgrading pip, urllib3 and httplib2 so that tempest tests can be run on openstack control node + # this needs to happen after devstack runs because it seems devstack is pulling in specific versions + # of these libs that are not working for tempest. + ${SSH} ${!CONTROLIP} "sudo pip install --upgrade pip" + ${SSH} ${!CONTROLIP} "sudo pip install urllib3 --upgrade" + ${SSH} ${!CONTROLIP} "sudo pip install httplib2 --upgrade" -# External Network -echo "prepare external networks by adding vxlan tunnels between all nodes on a separate bridge..." -devstack_index=1 -for ip in ${OPENSTACK_CONTROL_NODE_IP} ${COMPUTE_IPS[*]} -do - # FIXME - Workaround, ODL (new netvirt) currently adds PUBLIC_BRIDGE as a port in br-int since it doesn't see such a bridge existing when we stack - ${SSH} $ip "sudo ovs-vsctl --if-exists del-port br-int $PUBLIC_BRIDGE" - ${SSH} $ip "sudo ovs-vsctl --may-exist add-br $PUBLIC_BRIDGE -- set bridge $PUBLIC_BRIDGE other-config:disable-in-band=true other_config:hwaddr=f6:00:00:ff:01:0$((devstack_index++))" -done + # Gather Compute IPs for the site + for j in `seq 1 ${NUM_COMPUTES_PER_SITE}` + do + COMPUTE_INDEX=$(((i-1) * NUM_COMPUTES_PER_SITE + j)) + IP_VAR=OPENSTACK_COMPUTE_NODE_${COMPUTE_INDEX}_IP + COMPUTE_IPS[$((j-1))]=${!IP_VAR} + done -# ipsec support -if [ "${IPSEC_VXLAN_TUNNELS_ENABLED}" == "yes" ]; then - ALL_NODES=(${OPENSTACK_CONTROL_NODE_IP} ${COMPUTE_IPS[*]}) - for ((inx_ip1=0; inx_ip1<$((${#ALL_NODES[@]} - 1)); inx_ip1++)) + # Need to disable firewalld and iptables in compute nodes as well + for ip in ${COMPUTE_IPS[*]} do - for ((inx_ip2=$((inx_ip1 + 1)); inx_ip2<${#ALL_NODES[@]}; inx_ip2++)) - do - KEY1=0x$(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) - KEY2=0x$(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) - ID=0x$(dd if=/dev/urandom count=4 bs=1 2> /dev/null| xxd -p -c 8) - ip1=${ALL_NODES[$inx_ip1]} - ip2=${ALL_NODES[$inx_ip2]} - ${SSH} $ip1 "sudo ip xfrm state add src $ip1 dst $ip2 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" - ${SSH} $ip1 "sudo ip xfrm state add src $ip2 dst $ip1 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" - ${SSH} $ip1 "sudo ip xfrm policy add src $ip1 dst $ip2 proto udp dir out tmpl src $ip1 dst $ip2 proto esp reqid $ID mode transport" - ${SSH} $ip1 "sudo ip xfrm policy add src $ip2 dst $ip1 proto udp dir in tmpl src $ip2 dst $ip1 proto esp reqid $ID mode transport" - - ${SSH} $ip2 "sudo ip xfrm state add src $ip2 dst $ip1 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" - ${SSH} $ip2 "sudo ip xfrm state add src $ip1 dst $ip2 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" - ${SSH} $ip2 "sudo ip xfrm policy add src $ip2 dst $ip1 proto udp dir out tmpl src $ip2 dst $ip1 proto esp reqid $ID mode transport" - ${SSH} $ip2 "sudo ip xfrm policy add src $ip1 dst $ip2 proto udp dir in tmpl src $ip1 dst $ip2 proto esp reqid $ID mode transport" - done + scp ${WORKSPACE}/disable_firewall.sh "${ip}:/tmp" + ${SSH} "${ip}" "sudo bash /tmp/disable_firewall.sh" done - for ip in ${OPENSTACK_CONTROL_NODE_IP} ${COMPUTE_IPS[*]} + # External Network + echo "prepare external networks by adding vxlan tunnels between all nodes on a separate bridge..." + # FIXME Should there be a unique gateway IP and devstack index for each site? + devstack_index=1 + for ip in ${!CONTROLIP} ${COMPUTE_IPS[*]} do - echo "ip xfrm configuration for node $ip:" - ${SSH} $ip "sudo ip xfrm policy list" - ${SSH} $ip "sudo ip xfrm state list" + # FIXME - Workaround, ODL (new netvirt) currently adds PUBLIC_BRIDGE as a port in br-int since it doesn't see such a bridge existing when we stack + ${SSH} $ip "sudo ovs-vsctl --if-exists del-port br-int $PUBLIC_BRIDGE" + ${SSH} $ip "sudo ovs-vsctl --may-exist add-br $PUBLIC_BRIDGE -- set bridge $PUBLIC_BRIDGE other-config:disable-in-band=true other_config:hwaddr=f6:00:00:ff:01:0$((devstack_index++))" done -fi -# Control Node - PUBLIC_BRIDGE will act as the external router -GATEWAY_IP="10.10.10.250" # FIXME this should be a parameter, also shared with integration-test -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ip link add link ${PUBLIC_BRIDGE} name ${PUBLIC_BRIDGE}.167 type vlan id 167" -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ifconfig ${PUBLIC_BRIDGE} up" -${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ifconfig ${PUBLIC_BRIDGE}.167 up ${GATEWAY_IP}/24" -compute_index=1 -for compute_ip in ${COMPUTE_IPS[*]} -do - # Tunnel from controller to compute - PORT_NAME=compute$((compute_index++))_vxlan - ${SSH} ${OPENSTACK_CONTROL_NODE_IP} "sudo ovs-vsctl add-port $PUBLIC_BRIDGE $PORT_NAME -- set interface $PORT_NAME type=vxlan options:local_ip="${OPENSTACK_CONTROL_NODE_IP}" options:remote_ip="$compute_ip" options:dst_port=9876 options:key=flow" + # ipsec support + if [ "${IPSEC_VXLAN_TUNNELS_ENABLED}" == "yes" ]; then + ALL_NODES=(${!CONTROLIP} ${COMPUTE_IPS[*]}) + for ((inx_ip1=0; inx_ip1<$((${#ALL_NODES[@]} - 1)); inx_ip1++)) + do + for ((inx_ip2=$((inx_ip1 + 1)); inx_ip2<${#ALL_NODES[@]}; inx_ip2++)) + do + KEY1=0x$(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) + KEY2=0x$(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) + ID=0x$(dd if=/dev/urandom count=4 bs=1 2> /dev/null| xxd -p -c 8) + ip1=${ALL_NODES[$inx_ip1]} + ip2=${ALL_NODES[$inx_ip2]} + ${SSH} $ip1 "sudo ip xfrm state add src $ip1 dst $ip2 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" + ${SSH} $ip1 "sudo ip xfrm state add src $ip2 dst $ip1 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" + ${SSH} $ip1 "sudo ip xfrm policy add src $ip1 dst $ip2 proto udp dir out tmpl src $ip1 dst $ip2 proto esp reqid $ID mode transport" + ${SSH} $ip1 "sudo ip xfrm policy add src $ip2 dst $ip1 proto udp dir in tmpl src $ip2 dst $ip1 proto esp reqid $ID mode transport" + + ${SSH} $ip2 "sudo ip xfrm state add src $ip2 dst $ip1 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" + ${SSH} $ip2 "sudo ip xfrm state add src $ip1 dst $ip2 proto esp spi $ID reqid $ID mode transport auth sha256 $KEY1 enc aes $KEY2" + ${SSH} $ip2 "sudo ip xfrm policy add src $ip2 dst $ip1 proto udp dir out tmpl src $ip2 dst $ip1 proto esp reqid $ID mode transport" + ${SSH} $ip2 "sudo ip xfrm policy add src $ip1 dst $ip2 proto udp dir in tmpl src $ip1 dst $ip2 proto esp reqid $ID mode transport" + done + done + + for ip in ${!CONTROLIP} ${COMPUTE_IPS[*]} + do + echo "ip xfrm configuration for node $ip:" + ${SSH} $ip "sudo ip xfrm policy list" + ${SSH} $ip "sudo ip xfrm state list" + done + fi - # Tunnel from compute to controller - PORT_NAME=control_vxlan - ${SSH} ${compute_ip} "sudo ovs-vsctl add-port $PUBLIC_BRIDGE $PORT_NAME -- set interface $PORT_NAME type=vxlan options:local_ip="$compute_ip" options:remote_ip="${OPENSTACK_CONTROL_NODE_IP}" options:dst_port=9876 options:key=flow" + # Control Node - PUBLIC_BRIDGE will act as the external router + GATEWAY_IP="10.10.10.250" # FIXME this should be a parameter, also shared with integration-test + GATEWAY_VLAN_ID=167 + ${SSH} ${!CONTROLIP} "sudo ip link add link ${PUBLIC_BRIDGE} name ${PUBLIC_BRIDGE}.${GATEWAY_VLAN_ID} type vlan id ${GATEWAY_VLAN_ID}" + ${SSH} ${!CONTROLIP} "sudo ifconfig ${PUBLIC_BRIDGE} up" + ${SSH} ${!CONTROLIP} "sudo ifconfig ${PUBLIC_BRIDGE}.${GATEWAY_VLAN_ID} up ${GATEWAY_IP}/24" + + # Computes + compute_index=1 + for compute_ip in ${COMPUTE_IPS[*]} + do + # Tunnel from controller to compute + COMPUTEPORT=compute$(( compute_index++ ))_vxlan + ${SSH} ${!CONTROLIP} "sudo ovs-vsctl add-port $PUBLIC_BRIDGE $COMPUTEPORT -- set interface $COMPUTEPORT type=vxlan options:local_ip="${!CONTROLIP}" options:remote_ip="$compute_ip" options:dst_port=9876 options:key=flow" + # Tunnel from compute to controller + CONTROLPORT="control_vxlan" + ${SSH} $compute_ip "sudo ovs-vsctl add-port $PUBLIC_BRIDGE $CONTROLPORT -- set interface $CONTROLPORT type=vxlan options:local_ip="$compute_ip" options:remote_ip="${!CONTROLIP}" options:dst_port=9876 options:key=flow" + done done -if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then - HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM} - odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP - HA_PROXY_IP=${!odlmgrip} +if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then + odlmgrip=OPENSTACK_HAPROXY_1_IP + HA_PROXY_IP=${!odlmgrip} + HA_PROXY_1_IP=${!odlmgrip} + odlmgrip2=OPENSTACK_HAPROXY_2_IP + HA_PROXY_2_IP=${!odlmgrip2} + odlmgrip3=OPENSTACK_HAPROXY_1_IP + HA_PROXY_3_IP=${!odlmgrip3} else - HA_PROXY_IP=${ODL_SYSTEM_IP} + HA_PROXY_IP=${ODL_SYSTEM_IP} + HA_PROXY_1_IP=${ODL_SYSTEM_1_IP} + HA_PROXY_2_IP=${ODL_SYSTEM_2_IP} + HA_PROXY_3_IP=${ODL_SYSTEM_3_IP} fi + echo "Locating test plan to use..." testplan_filepath="${WORKSPACE}/test/csit/testplans/${STREAMTESTPLAN}" if [ ! -f "${testplan_filepath}" ]; then @@ -738,6 +755,7 @@ cat testplan.txt SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '` +#FIXME currently support only 1 site echo "Starting Robot test suites ${SUITES} ..." # please add pybot -v arguments on a single line and alphabetized pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude \ @@ -746,9 +764,13 @@ pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude \ -v CONTROLLER_USER:${USER} \ -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack \ -v HA_PROXY_IP:${HA_PROXY_IP} \ + -v HA_PROXY_1_IP:${HA_PROXY_1_IP} \ + -v HA_PROXY_2_IP:${HA_PROXY_2_IP} \ + -v HA_PROXY_3_IP:${HA_PROXY_3_IP} \ -v JDKVERSION:${JDKVERSION} \ -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} \ -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} \ + -v NUM_OPENSTACK_SITES:${NUM_OPENSTACK_SITES} \ -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} \ -v NUM_TOOLS_SYSTEM:${NUM_TOOLS_SYSTEM} \ -v ODL_STREAM:${DISTROSTREAM} \ @@ -756,10 +778,23 @@ pybot -N ${TESTPLAN} --removekeywords wuks -c critical -e exclude \ -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} \ -v ODL_SYSTEM_2_IP:${ODL_SYSTEM_2_IP} \ -v ODL_SYSTEM_3_IP:${ODL_SYSTEM_3_IP} \ - -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \ + -v ODL_SYSTEM_4_IP:${ODL_SYSTEM_4_IP} \ + -v ODL_SYSTEM_5_IP:${ODL_SYSTEM_5_IP} \ + -v ODL_SYSTEM_6_IP:${ODL_SYSTEM_6_IP} \ + -v ODL_SYSTEM_7_IP:${ODL_SYSTEM_7_IP} \ + -v ODL_SYSTEM_8_IP:${ODL_SYSTEM_8_IP} \ + -v ODL_SYSTEM_9_IP:${ODL_SYSTEM_9_IP} \ + -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_1_IP} \ + -v OS_CONTROL_NODE_1_IP:${OPENSTACK_CONTROL_NODE_1_IP} \ + -v OS_CONTROL_NODE_2_IP:${OPENSTACK_CONTROL_NODE_2_IP} \ + -v OS_CONTROL_NODE_3_IP:${OPENSTACK_CONTROL_NODE_3_IP} \ -v OPENSTACK_BRANCH:${OPENSTACK_BRANCH} \ -v OS_COMPUTE_1_IP:${OPENSTACK_COMPUTE_NODE_1_IP} \ -v OS_COMPUTE_2_IP:${OPENSTACK_COMPUTE_NODE_2_IP} \ + -v OS_COMPUTE_3_IP:${OPENSTACK_COMPUTE_NODE_3_IP} \ + -v OS_COMPUTE_4_IP:${OPENSTACK_COMPUTE_NODE_4_IP} \ + -v OS_COMPUTE_5_IP:${OPENSTACK_COMPUTE_NODE_5_IP} \ + -v OS_COMPUTE_6_IP:${OPENSTACK_COMPUTE_NODE_6_IP} \ -v OS_USER:${USER} \ -v PUBLIC_PHYSICAL_NETWORK:${PUBLIC_PHYSICAL_NETWORK} \ -v SECURITY_GROUP_MODE:${SECURITY_GROUP_MODE} \ diff --git a/jjb/integration/include-raw-integration-get-slave-addresses.sh b/jjb/integration/include-raw-integration-get-slave-addresses.sh index eb58dca52..d41e88dfc 100644 --- a/jjb/integration/include-raw-integration-get-slave-addresses.sh +++ b/jjb/integration/include-raw-integration-get-slave-addresses.sh @@ -3,6 +3,7 @@ ODL_SYSTEM=() TOOLS_SYSTEM=() OPENSTACK_SYSTEM=() +[ "$NUM_OPENSTACK_SITES" ] || NUM_OPENSTACK_SITES=1 source $WORKSPACE/.venv-openstack/bin/activate ADDR=(`openstack stack show -f json -c outputs $STACK_NAME | \ @@ -30,10 +31,12 @@ echo "NUM_ODL_SYSTEM=${#ODL_SYSTEM[@]}" >> slave_addresses.txt echo "NUM_TOOLS_SYSTEM=${#TOOLS_SYSTEM[@]}" >> slave_addresses.txt #if HA Proxy is requested the last devstack node will be configured as haproxy if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then - echo "NUM_OPENSTACK_SYSTEM=$(( ${#OPENSTACK_SYSTEM[@]} - 1 ))" >> slave_addresses.txt + # HA Proxy is installed on one OPENSTACK_SYSTEM VM on each site + NUM_OPENSTACK_SYSTEM=$(( ${#OPENSTACK_SYSTEM[@]} - ${NUM_OPENSTACK_SITES} )) else - echo "NUM_OPENSTACK_SYSTEM=${#OPENSTACK_SYSTEM[@]}" >> slave_addresses.txt + NUM_OPENSTACK_SYSTEM=${#OPENSTACK_SYSTEM[@]} fi +echo "NUM_OPENSTACK_SYSTEM=${NUM_OPENSTACK_SYSTEM}" >> slave_addresses.txt # Add alias for ODL_SYSTEM_1_IP as ODL_SYSTEM_IP echo "ODL_SYSTEM_IP=${ODL_SYSTEM[0]}" >> slave_addresses.txt @@ -49,9 +52,28 @@ do echo "TOOLS_SYSTEM_$((i+1))_IP=${TOOLS_SYSTEM[${i}]}" >> slave_addresses.txt done -echo "OPENSTACK_CONTROL_NODE_IP=${OPENSTACK_SYSTEM[0]}" >> slave_addresses.txt -for i in `seq 1 $(( ${#OPENSTACK_SYSTEM[@]} - 1 ))` +openstack_index=0 +# Assuming number of openstack control nodes equals number of openstack sites +NUM_OPENSTACK_CONTROL_NODES=$(( NUM_OPENSTACK_SITES )) +echo "NUM_OPENSTACK_CONTROL_NODES=${NUM_OPENSTACK_CONTROL_NODES}" >> slave_addresses.txt +for i in `seq 0 $((NUM_OPENSTACK_CONTROL_NODES - 1))` do - echo "OPENSTACK_COMPUTE_NODE_$((i))_IP=${OPENSTACK_SYSTEM[${i}]}" >> slave_addresses.txt + echo "OPENSTACK_CONTROL_NODE_$((i+1))_IP=${OPENSTACK_SYSTEM[$((openstack_index++))]}" >> slave_addresses.txt +done + +# The rest of the openstack nodes until NUM_OPENSTACK_SYSTEM are computes +NUM_OPENSTACK_COMPUTE_NODES=$(( NUM_OPENSTACK_SYSTEM - NUM_OPENSTACK_CONTROL_NODES )) +echo "NUM_OPENSTACK_COMPUTE_NODES=${NUM_OPENSTACK_COMPUTE_NODES}" >> slave_addresses.txt +for i in `seq 0 $((NUM_OPENSTACK_COMPUTE_NODES - 1))` +do + echo "OPENSTACK_COMPUTE_NODE_$((i+1))_IP=${OPENSTACK_SYSTEM[$((openstack_index++))]}" >> slave_addresses.txt +done + +# The remaining openstack nodes are haproxy nodes (for ODL cluster) +NUM_OPENSTACK_HAPROXY_NODES=$(( ${#OPENSTACK_SYSTEM[@]} - NUM_OPENSTACK_SYSTEM )) +echo "NUM_OPENSTACK_HAPROXY_NODES=${NUM_OPENSTACK_HAPROXY_NODES}" >> slave_addresses.txt +for i in `seq 0 $((NUM_OPENSTACK_HAPROXY_NODES - 1))` +do + echo "OPENSTACK_HAPROXY_$((i+1))_IP=${OPENSTACK_SYSTEM[$((openstack_index++))]}" >> slave_addresses.txt done # vim: sw=4 ts=4 sts=4 et ft=sh : diff --git a/jjb/integration/integration-templates.yaml b/jjb/integration/integration-templates.yaml index 69795b620..54231f87c 100644 --- a/jjb/integration/integration-templates.yaml +++ b/jjb/integration/integration-templates.yaml @@ -1272,6 +1272,11 @@ default: '{lbaas-service-provider}' description: 'The NEUTRON_LBAAS_SERVICE_PROVIDERV2 value to be used in local.conf - only relevant when using neutron-lbaas' + - string: + name: NUM_OPENSTACK_SITES + default: '{num-openstack-sites}' + description: 'Number of openstack sites' + scm: - integration-gerrit-scm: basedir: 'test' diff --git a/jjb/netvirt/netvirt-csit-multi-openstack.yaml b/jjb/netvirt/netvirt-csit-multi-openstack.yaml index f24e1aaf2..cac427f40 100644 --- a/jjb/netvirt/netvirt-csit-multi-openstack.yaml +++ b/jjb/netvirt/netvirt-csit-multi-openstack.yaml @@ -72,3 +72,51 @@ openstack_system_image: 'CentOS 7 - devstack-newton - 20170314-2256' security-group-mode: '{sg-mode}' + +- project: + name: netvirt-csit-multi-site-openstack-integration + jobs: + - '{project}-csit-{topology}-openstack-{openstack}-{functionality}-{stream}' + + project: 'netvirt' + + topology: + - 2site-1node: + openstack_system_count: 6 + odl_system_count: 2 + enable-haproxy: 'no' + install-features: 'odl-netvirt-openstack,odl-netvirt-federation' + num-openstack-sites: 2 + - 2site-3node: + openstack_system_count: 8 + odl_system_count: 6 + enable-haproxy: 'yes' + install-features: 'odl-jolokia,odl-netvirt-openstack,odl-netvirt-federation' + num-openstack-sites: 2 + + testplan: '{project}-{topology}-openstack.txt' + + functionality: + - upstream-stateful: + schedule: 'H H * * *' + sg-mode: 'stateful' + - upstream-learn: + schedule: 'H H * * *' + sg-mode: 'learn' + + stream: + - carbon: + branch: 'master' + jre: 'openjdk8' + - boron: + branch: 'stable/boron' + jre: 'openjdk8' + + openstack: + - newton-nodl-v2: + openstack-branch: 'stable/newton' + odl-ml2-branch: 'stable/newton' + odl-ml2-driver-version: 'v2' + openstack_system_image: 'CentOS 7 - devstack-newton - 20170314-2256' + + security-group-mode: '{sg-mode}' diff --git a/jjb/releng-defaults.yaml b/jjb/releng-defaults.yaml index 67a73bb25..f14d610a3 100644 --- a/jjb/releng-defaults.yaml +++ b/jjb/releng-defaults.yaml @@ -9,6 +9,7 @@ # Timeout in minutes build-timeout: 360 build-node: centos7-java-builder-2c-8g + num-openstack-sites: 1 email-recipients: 'jenkins@lists.opendaylight.org' odl-ml2-driver-repo: 'https://github.com/openstack/networking-odl' -- 2.36.6