jjb/integration/include-raw-integration-deploy-openstack-run-test.sh

   1 #@IgnoreInspection BashAddShebang
   2 # Activate robotframework virtualenv
   3 # ${ROBOT_VENV} comes from the include-raw-integration-install-robotframework.sh
   4 # script.
   5 source ${ROBOT_VENV}/bin/activate
   6
   7 echo "#################################################"
   8 echo "##         Deploy Openstack 3-node             ##"
   9 echo "#################################################"
  10
  11 function create_control_node_local_conf {
  12 local_conf_file_name=${WORKSPACE}/local.conf_control
  13 cat > ${local_conf_file_name} << EOF
  14 [[local|localrc]]
  15 LOGFILE=stack.sh.log
  16 SCREEN_LOGDIR=/opt/stack/data/log
  17 LOG_COLOR=False
  18 RECLONE=yes
  19 EOF
  20
  21 IFS=,
  22 for service_name in ${DISABLE_OS_SERVICES}
  23 do
  24 cat >> ${local_conf_file_name} << EOF
  25 disable_service ${service_name}
  26 EOF
  27 done
  28 for service_name in ${ENABLE_OS_SERVICES}
  29 do
  30 cat >> ${local_conf_file_name} << EOF
  31 enable_service ${service_name}
  32 EOF
  33 done
  34 unset IFS
  35
  36 cat >> ${local_conf_file_name} << EOF
  37 HOST_IP=$OPENSTACK_CONTROL_NODE_IP
  38 SERVICE_HOST=\$HOST_IP
  39
  40 NEUTRON_CREATE_INITIAL_NETWORKS=False
  41 Q_PLUGIN=ml2
  42 Q_ML2_TENANT_NETWORK_TYPE=${TENANT_NETWORK_TYPE}
  43 Q_OVS_USE_VETH=True
  44
  45 ENABLE_TENANT_TUNNELS=True
  46
  47 MYSQL_HOST=\$SERVICE_HOST
  48 RABBIT_HOST=\$SERVICE_HOST
  49 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
  50 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
  51 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
  52
  53 MYSQL_PASSWORD=mysql
  54 RABBIT_PASSWORD=rabbit
  55 SERVICE_TOKEN=service
  56 SERVICE_PASSWORD=admin
  57 ADMIN_PASSWORD=admin
  58
  59 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_VERSION}
  60
  61 ODL_PORT=8080
  62 ODL_MODE=externalodl
  63 LIBVIRT_TYPE=qemu
  64
  65 EOF
  66
  67 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
  68 odl_list=${ODL_SYSTEM_1_IP}
  69 for i in `seq 2 ${NUM_ODL_SYSTEM}`
  70 do
  71 odlip=ODL_SYSTEM_${i}_IP
  72 odl_list=${odl_list},${!odlip}
  73 done
  74 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
  75 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
  76 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
  77 odl_mgr_ip=${!odlmgrip}
  78 else
  79 odl_mgr_ip=${ODL_SYSTEM_1_IP}
  80 fi
  81 cat >> ${local_conf_file_name} << EOF
  82 ODL_OVS_MANAGERS=${odl_list}
  83 ODL_MGR_IP=${odl_mgr_ip}
  84 EOF
  85 else
  86 cat >> ${local_conf_file_name} << EOF
  87 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
  88 EOF
  89 fi
  90
  91 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
  92 cat >> ${local_conf_file_name} << EOF
  93
  94 PUBLIC_BRIDGE=${PUBLIC_BRIDGE}
  95 ODL_PROVIDER_MAPPINGS=${PUBLIC_BRIDGE}:br100
  96
  97 disable_service q-l3
  98 Q_L3_ENABLED=True
  99 ODL_L3=True
 100 PUBLIC_INTERFACE=br100
 101 [[post-config|\$NEUTRON_CONF]]
 102 [DEFAULT]
 103 service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
 104
 105 EOF
 106 fi
 107 cat >> ${local_conf_file_name} << EOF
 108 [[post-config|/etc/neutron/plugins/ml2/ml2_conf.ini]]
 109 [agent]
 110 minimize_polling=True
 111
 112 [[post-config|/etc/neutron/dhcp_agent.ini]]
 113 [DEFAULT]
 114 force_metadata = True
 115 enable_isolated_metadata = True
 116
 117 [[post-config|/etc/nova/nova.conf]]
 118 [DEFAULT]
 119 force_config_drive = False
 120
 121 EOF
 122
 123 echo "local.conf Created...."
 124 cat ${local_conf_file_name}
 125 }
 126
 127 function create_compute_node_local_conf {
 128 HOSTIP=$1
 129 local_conf_file_name=${WORKSPACE}/local.conf_compute_${HOSTIP}
 130 cat > ${local_conf_file_name} << EOF
 131 [[local|localrc]]
 132 LOGFILE=stack.sh.log
 133 LOG_COLOR=False
 134 SCREEN_LOGDIR=/opt/stack/data/log
 135 RECLONE=yes
 136
 137 NOVA_VNC_ENABLED=True
 138 MULTI_HOST=1
 139 ENABLED_SERVICES=n-cpu
 140
 141 HOST_IP=${HOSTIP}
 142 SERVICE_HOST=${OPENSTACK_CONTROL_NODE_IP}
 143
 144 Q_PLUGIN=ml2
 145 ENABLE_TENANT_TUNNELS=True
 146 Q_ML2_TENANT_NETWORK_TYPE=vxlan
 147
 148 Q_HOST=\$SERVICE_HOST
 149 MYSQL_HOST=\$SERVICE_HOST
 150 RABBIT_HOST=\$SERVICE_HOST
 151 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
 152 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
 153 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
 154
 155 MYSQL_PASSWORD=mysql
 156 RABBIT_PASSWORD=rabbit
 157 SERVICE_TOKEN=service
 158 SERVICE_PASSWORD=admin
 159 ADMIN_PASSWORD=admin
 160
 161 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_VERSION}
 162 ODL_MODE=compute
 163 LIBVIRT_TYPE=qemu
 164
 165 EOF
 166
 167 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
 168 odl_list=${ODL_SYSTEM_1_IP}
 169 for i in `seq 2 ${NUM_ODL_SYSTEM}`
 170 do
 171 odlip=ODL_SYSTEM_${i}_IP
 172 odl_list=${odl_list},${!odlip}
 173 done
 174 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 175 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 176 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 177 odl_mgr_ip=${!odlmgrip}
 178 else
 179 odl_mgr_ip=${ODL_SYSTEM_1_IP}
 180 fi
 181 cat >> ${local_conf_file_name} << EOF
 182 ODL_OVS_MANAGERS=${odl_list}
 183 ODL_MGR_IP=${odl_mgr_ip}
 184 EOF
 185 else
 186 cat >> ${local_conf_file_name} << EOF
 187 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
 188 EOF
 189 fi
 190
 191 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
 192 cat >> ${local_conf_file_name} << EOF
 193 # Uncomment lines below if odl-compute is to be used for l3 forwarding
 194 Q_L3_ENABLED=True
 195 ODL_L3=True
 196 PUBLIC_INTERFACE=br100
 197 EOF
 198 fi
 199 echo "local.conf Created...."
 200 cat ${local_conf_file_name}
 201 }
 202
 203 function configure_haproxy_for_neutron_requests () {
 204 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 205 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 206 ha_proxy_ip=${!odlmgrip}
 207
 208 cat > ${WORKSPACE}/install_ha_proxy.sh<< EOF
 209 sudo systemctl stop firewalld
 210 sudo yum -y install policycoreutils-python haproxy
 211 EOF
 212
 213 cat > ${WORKSPACE}/haproxy.cfg << EOF
 214 global
 215   daemon
 216   group  haproxy
 217   log  /dev/log local0
 218   maxconn  20480
 219   pidfile  /tmp/haproxy.pid
 220   user  haproxy
 221
 222 defaults
 223   log  global
 224   maxconn  4096
 225   mode  tcp
 226   retries  3
 227   timeout  http-request 10s
 228   timeout  queue 1m
 229   timeout  connect 10s
 230   timeout  client 1m
 231   timeout  server 1m
 232   timeout  check 10s
 233
 234 listen opendaylight
 235   bind ${ha_proxy_ip}:8080
 236   balance source
 237 EOF
 238
 239 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 240 do
 241 odlip=ODL_SYSTEM_${i}_IP
 242 cat >> ${WORKSPACE}/haproxy.cfg << EOF
 243   server controller-$i ${!odlip}:8080 check fall 5 inter 2000 rise 2
 244 EOF
 245 done
 246
 247 cat > ${WORKSPACE}/deploy_ha_proxy.sh<< EOF
 248 sudo chown haproxy:haproxy /tmp/haproxy.cfg
 249 sudo sed -i 's/\\/etc\\/haproxy\\/haproxy.cfg/\\/tmp\\/haproxy.cfg/g' /usr/lib/systemd/system/haproxy.service
 250 sudo /usr/sbin/semanage permissive -a haproxy_t
 251 sudo systemctl restart haproxy
 252 sleep 3
 253 sudo netstat -tunpl
 254 sudo systemctl status haproxy
 255 true
 256 EOF
 257 scp ${WORKSPACE}/install_ha_proxy.sh ${ha_proxy_ip}:/tmp
 258 ssh ${ha_proxy_ip} "sudo bash /tmp/install_ha_proxy.sh"
 259 scp ${WORKSPACE}/haproxy.cfg ${ha_proxy_ip}:/tmp
 260 scp ${WORKSPACE}/deploy_ha_proxy.sh ${ha_proxy_ip}:/tmp
 261 ssh ${ha_proxy_ip} "sudo bash /tmp/deploy_ha_proxy.sh"
 262 }
 263
 264 function collect_logs_and_exit (){
 265 set +e  # We do not want to create red dot just because something went wrong while fetching logs.
 266 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 267 do
 268     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 269     echo "killing karaf process..."
 270     ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
 271 done
 272
 273 sleep 5
 274 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 275 do
 276     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 277     ssh "${!CONTROLLERIP}"  "cp -r /tmp/${BUNDLEFOLDER}/data/log /tmp/odl_log"
 278     ssh "${!CONTROLLERIP}"  "tar -cf /tmp/odl${i}_karaf.log.tar /tmp/odl_log/*"
 279     scp "${!CONTROLLERIP}:/tmp/odl${i}_karaf.log.tar" "${WORKSPACE}/odl${i}_karaf.log.tar"
 280     tar -xvf ${WORKSPACE}/odl${i}_karaf.log.tar -C . --strip-components 2 --transform s/karaf/odl${i}_karaf/g
 281     rm ${WORKSPACE}/odl${i}_karaf.log.tar
 282 done
 283
 284 scp ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/nohup.out "openstack_control_stack.log"
 285 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 286 do
 287     OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 288     scp "${!OSIP}:/opt/stack/devstack/nohup.out" "openstack_compute_stack_${i}.log"
 289 done
 290 }
 291
 292 cat > ${WORKSPACE}/disable_firewall.sh << EOF
 293 sudo systemctl stop firewalld
 294 sudo systemctl stop iptables
 295 true
 296 EOF
 297
 298 cat > ${WORKSPACE}/get_devstack.sh << EOF
 299 sudo systemctl stop firewalld
 300 sudo yum install bridge-utils -y
 301 sudo systemctl stop  NetworkManager
 302 #Disable NetworkManager and kill dhclient and dnsmasq
 303 sudo systemctl stop NetworkManager
 304 sudo killall dhclient
 305 sudo killall dnsmasq
 306 #Workaround for mysql failure
 307 echo "127.0.0.1    localhost \${HOSTNAME}" > /tmp/hosts
 308 echo "::1   localhost  \${HOSTNAME}" >> /tmp/hosts
 309 sudo mv /tmp/hosts /etc/hosts
 310 sudo /usr/sbin/brctl addbr br100
 311 sudo ifconfig eth0 mtu 2000
 312 sudo mkdir /opt/stack
 313 sudo chmod 777 /opt/stack
 314 cd /opt/stack
 315 git clone https://git.openstack.org/openstack-dev/devstack
 316 cd devstack
 317 git checkout $OPENSTACK_BRANCH
 318 EOF
 319
 320 echo "Create HAProxy if needed"
 321 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 322  echo "Need to configure HAProxy"
 323  configure_haproxy_for_neutron_requests
 324 fi
 325
 326 os_node_list=()
 327 echo "Stack the Control Node"
 328 scp ${WORKSPACE}/get_devstack.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp
 329 ssh ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/get_devstack.sh"
 330 create_control_node_local_conf
 331 scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf
 332 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 333 ssh ${OPENSTACK_CONTROL_NODE_IP} "ps -ef | grep stack.sh"
 334 ssh ${OPENSTACK_CONTROL_NODE_IP} "ls -lrt /opt/stack/devstack/nohup.out"
 335 os_node_list+=(${OPENSTACK_CONTROL_NODE_IP})
 336
 337
 338 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 339 do
 340     COMPUTEIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 341     scp ${WORKSPACE}/get_devstack.sh  ${!COMPUTEIP}:/tmp
 342     ssh ${!COMPUTEIP} "bash /tmp/get_devstack.sh"
 343     create_compute_node_local_conf ${!COMPUTEIP}
 344     scp ${WORKSPACE}/local.conf_compute_${!COMPUTEIP} ${!COMPUTEIP}:/opt/stack/devstack/local.conf
 345     ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 346     ssh ${!COMPUTEIP} "ps -ef | grep stack.sh"
 347     os_node_list+=(${!COMPUTEIP})
 348 done
 349
 350 cat > ${WORKSPACE}/check_stacking.sh << EOF
 351 > /tmp/stack_progress
 352 ps -ef | grep "stack.sh" | grep -v grep
 353 ret=\$?
 354 if [ \${ret} -eq 1 ]; then
 355   grep "This is your host IP address:" /opt/stack/devstack/nohup.out
 356   if [ \$? -eq 0 ]; then
 357      echo "Stacking Complete" > /tmp/stack_progress
 358   else
 359      echo "Stacking Failed" > /tmp/stack_progress
 360   fi
 361 elif [ \${ret} -eq 0 ]; then
 362   echo "Still Stacking" > /tmp/stack_progress
 363 fi
 364 EOF
 365
 366 #the checking is repeated for an hour
 367 iteration=0
 368 in_progress=1
 369 while [ ${in_progress} -eq 1 ]; do
 370 iterator=$(($iterator + 1))
 371 for index in ${!os_node_list[@]}
 372 do
 373 echo "Check the status of stacking in ${os_node_list[index]}"
 374 scp ${WORKSPACE}/check_stacking.sh  ${os_node_list[index]}:/tmp
 375 ssh ${os_node_list[index]} "bash /tmp/check_stacking.sh"
 376 scp ${os_node_list[index]}:/tmp/stack_progress .
 377 #debug
 378 cat stack_progress
 379 stacking_status=`cat stack_progress`
 380 if [ "$stacking_status" == "Still Stacking" ]; then
 381   continue
 382 elif [ "$stacking_status" == "Stacking Failed" ]; then
 383   collect_logs_and_exit
 384   exit 1
 385 elif [ "$stacking_status" == "Stacking Complete" ]; then
 386   unset os_node_list[index]
 387   if  [ ${#os_node_list[@]} -eq 0 ]; then
 388      in_progress=0
 389   fi
 390 fi
 391 done
 392  echo "sleep for a minute before the next check"
 393  sleep 60
 394  if [ ${iteration} -eq 60 ]; then
 395   collect_logs_and_exit
 396   exit 1
 397  fi
 398 done
 399
 400 #Need to disable firewalld and iptables in control node
 401 echo "Stop Firewall in Control Node for compute nodes to be able to reach the ports and add to hypervisor-list"
 402 scp ${WORKSPACE}/disable_firewall.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp
 403 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo bash /tmp/disable_firewall.sh"
 404 echo "sleep for a minute and print hypervisor-list"
 405 sleep 60
 406 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; source openrc admin admin; nova hypervisor-list;nova-manage service list"
 407
 408 #Need to disable firewalld and iptables in compute nodes as well
 409 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 410 do
 411     OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 412     scp ${WORKSPACE}/disable_firewall.sh "${!OSIP}:/tmp"
 413     ssh "${!OSIP}" "sudo bash /tmp/disable_firewall.sh"
 414 done
 415
 416 # upgrading pip, urllib3 and httplib2 so that tempest tests can be run on ${OPENSTACK_CONTROL_NODE_IP}
 417 # this needs to happen after devstack runs because it seems devstack is pulling in specific versions
 418 # of these libs that are not working for tempest.
 419 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install --upgrade pip"
 420 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install urllib3 --upgrade"
 421 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install httplib2 --upgrade"
 422
 423 echo "Locating test plan to use..."
 424 testplan_filepath="${WORKSPACE}/test/csit/testplans/${STREAMTESTPLAN}"
 425 if [ ! -f "${testplan_filepath}" ]; then
 426     testplan_filepath="${WORKSPACE}/test/csit/testplans/${TESTPLAN}"
 427 fi
 428
 429 echo "Changing the testplan path..."
 430 cat "${testplan_filepath}" | sed "s:integration:${WORKSPACE}:" > testplan.txt
 431 cat testplan.txt
 432
 433 SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
 434
 435 echo "Starting Robot test suites ${SUITES} ..."
 436 pybot -N ${TESTPLAN} -c critical -e exclude -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp -v BUNDLE_URL:${ACTUALBUNDLEURL} \
 437 -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} -v JDKVERSION:${JDKVERSION} -v ODL_STREAM:${DISTROSTREAM} \
 438 -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} -v ODL_SYSTEM_2_IP:${ODL_SYSTEM_2_IP} \
 439 -v ODL_SYSTEM_3_IP:${ODL_SYSTEM_3_IP} -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} -v CONTROLLER_USER:${USER} -v OS_USER:${USER} \
 440 -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \
 441 -v OS_COMPUTE_1_IP:${OPENSTACK_COMPUTE_NODE_1_IP} -v OS_COMPUTE_2_IP:${OPENSTACK_COMPUTE_NODE_2_IP} \
 442 -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack -v USER_HOME:${HOME} ${TESTOPTIONS} ${SUITES} || true
 443
 444 echo "Tests Executed"
 445 DEVSTACK_TEMPEST_DIR="/opt/stack/tempest"
 446 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/subunit-1to2 < ${DEVSTACK_TEMPEST_DIR}/.testrepository/0 > ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt'"
 447 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo sh -c '${DEVSTACK_TEMPEST_DIR}/.tox/tempest/bin/python ${DEVSTACK_TEMPEST_DIR}/.tox/tempest/lib/python2.7/site-packages/os_testr/subunit2html.py ${DEVSTACK_TEMPEST_DIR}/subunit_log.txt ${DEVSTACK_TEMPEST_DIR}/tempest_results.html'"
 448 scp ${OPENSTACK_CONTROL_NODE_IP}:${DEVSTACK_TEMPEST_DIR}/tempest_results.html ${WORKSPACE}/
 449 collect_logs_and_exit
 450
 451 true  # perhaps Jenkins is testing last exit code
 452 # vim: ts=4 sw=4 sts=4 et ft=sh :