jjb/integration/include-raw-integration-deploy-openstack-run-test.sh

   1 #@IgnoreInspection BashAddShebang
   2 # Activate robotframework virtualenv
   3 # ${ROBOT_VENV} comes from the include-raw-integration-install-robotframework.sh
   4 # script.
   5 source ${ROBOT_VENV}/bin/activate
   6
   7 echo "#################################################"
   8 echo "##         Deploy Openstack 3-node             ##"
   9 echo "#################################################"
  10
  11 function create_control_node_local_conf {
  12 local_conf_file_name=${WORKSPACE}/local.conf_control
  13 cat > ${local_conf_file_name} << EOF
  14 [[local|localrc]]
  15 LOGFILE=stack.sh.log
  16 SCREEN_LOGDIR=/opt/stack/data/log
  17 LOG_COLOR=False
  18 RECLONE=yes
  19
  20 disable_service swift
  21 disable_service cinder
  22 disable_service n-net
  23 disable_service q-vpn
  24 enable_service q-svc
  25 enable_service q-dhcp
  26 enable_service q-meta
  27 enable_service tempest
  28 enable_service n-novnc
  29 enable_service n-cauth
  30
  31 HOST_IP=$OPENSTACK_CONTROL_NODE_IP
  32 SERVICE_HOST=\$HOST_IP
  33
  34 NEUTRON_CREATE_INITIAL_NETWORKS=False
  35 Q_PLUGIN=ml2
  36 Q_ML2_TENANT_NETWORK_TYPE=vxlan
  37
  38 ENABLE_TENANT_TUNNELS=True
  39
  40
  41 MYSQL_HOST=\$SERVICE_HOST
  42 RABBIT_HOST=\$SERVICE_HOST
  43 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
  44 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
  45 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
  46
  47 MYSQL_PASSWORD=mysql
  48 RABBIT_PASSWORD=rabbit
  49 SERVICE_TOKEN=service
  50 SERVICE_PASSWORD=admin
  51 ADMIN_PASSWORD=admin
  52
  53 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
  54
  55 ODL_PORT=8080
  56 ODL_MODE=externalodl
  57
  58 EOF
  59
  60 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
  61 odl_list=${ODL_SYSTEM_1_IP}
  62 for i in `seq 2 ${NUM_ODL_SYSTEM}`
  63 do
  64 odlip=ODL_SYSTEM_${i}_IP
  65 odl_list=${odl_list},${!odlip}
  66 done
  67 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
  68 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
  69 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
  70 odl_mgr_ip=${!odlmgrip}
  71 else
  72 odl_mgr_ip=${ODL_SYSTEM_1_IP}
  73 fi
  74 cat >> ${local_conf_file_name} << EOF
  75 ODL_OVS_MANAGERS=${odl_list}
  76 ODL_MGR_IP=${odl_mgr_ip}
  77 EOF
  78 else
  79 cat >> ${local_conf_file_name} << EOF
  80 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
  81 EOF
  82 fi
  83
  84 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
  85 cat >> ${local_conf_file_name} << EOF
  86
  87 ODL_PROVIDER_MAPPINGS=br-ex:br100
  88
  89 disable_service q-l3
  90 Q_L3_ENABLED=True
  91 ODL_L3=True
  92 PUBLIC_INTERFACE=br100
  93 [[post-config|\$NEUTRON_CONF]]
  94 [DEFAULT]
  95 service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
  96
  97 EOF
  98 fi
  99 cat >> ${local_conf_file_name} << EOF
 100 [[post-config|/etc/neutron/plugins/ml2/ml2_conf.ini]]
 101 [agent]
 102 minimize_polling=True
 103
 104 [[post-config|/etc/neutron/dhcp_agent.ini]]
 105 [DEFAULT]
 106 force_metadata = True
 107 enable_isolated_metadata = True
 108
 109 [[post-config|/etc/nova/nova.conf]]
 110 [DEFAULT]
 111 force_config_drive = False
 112
 113 EOF
 114
 115 echo "local.conf Created...."
 116 cat ${local_conf_file_name}
 117 }
 118
 119 function create_compute_node_local_conf {
 120 HOSTIP=$1
 121 local_conf_file_name=${WORKSPACE}/local.conf_compute_${HOSTIP}
 122 cat > ${local_conf_file_name} << EOF
 123 [[local|localrc]]
 124 LOGFILE=stack.sh.log
 125 LOG_COLOR=False
 126 SCREEN_LOGDIR=/opt/stack/data/log
 127 RECLONE=yes
 128
 129 NOVA_VNC_ENABLED=True
 130 MULTI_HOST=1
 131 ENABLED_SERVICES=n-cpu
 132
 133 HOST_IP=${HOSTIP}
 134 SERVICE_HOST=${OPENSTACK_CONTROL_NODE_IP}
 135
 136 Q_PLUGIN=ml2
 137 ENABLE_TENANT_TUNNELS=True
 138 Q_ML2_TENANT_NETWORK_TYPE=vxlan
 139
 140 Q_HOST=\$SERVICE_HOST
 141 MYSQL_HOST=\$SERVICE_HOST
 142 RABBIT_HOST=\$SERVICE_HOST
 143 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
 144 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
 145 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
 146
 147 MYSQL_PASSWORD=mysql
 148 RABBIT_PASSWORD=rabbit
 149 SERVICE_TOKEN=service
 150 SERVICE_PASSWORD=admin
 151 ADMIN_PASSWORD=admin
 152
 153 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
 154 ODL_MODE=compute
 155
 156 EOF
 157
 158 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
 159 odl_list=${ODL_SYSTEM_1_IP}
 160 for i in `seq 2 ${NUM_ODL_SYSTEM}`
 161 do
 162 odlip=ODL_SYSTEM_${i}_IP
 163 odl_list=${odl_list},${!odlip}
 164 done
 165 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 166 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 167 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 168 odl_mgr_ip=${!odlmgrip}
 169 else
 170 odl_mgr_ip=${ODL_SYSTEM_1_IP}
 171 fi
 172 cat >> ${local_conf_file_name} << EOF
 173 ODL_OVS_MANAGERS=${odl_list}
 174 ODL_MGR_IP=${odl_mgr_ip}
 175 EOF
 176 else
 177 cat >> ${local_conf_file_name} << EOF
 178 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
 179 EOF
 180 fi
 181
 182 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
 183 cat >> ${local_conf_file_name} << EOF
 184 # Uncomment lines below if odl-compute is to be used for l3 forwarding
 185 Q_L3_ENABLED=True
 186 ODL_L3=True
 187 PUBLIC_INTERFACE=br100
 188 EOF
 189 fi
 190 echo "local.conf Created...."
 191 cat ${local_conf_file_name}
 192 }
 193
 194 function configure_haproxy_for_neutron_requests () {
 195 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 196 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 197 ha_proxy_ip=${!odlmgrip}
 198
 199 cat > ${WORKSPACE}/install_ha_proxy.sh<< EOF
 200 sudo systemctl stop firewalld
 201 sudo yum -y install policycoreutils-python haproxy
 202 EOF
 203
 204 cat > ${WORKSPACE}/haproxy.cfg << EOF
 205 global
 206   daemon
 207   group  haproxy
 208   log  /dev/log local0
 209   maxconn  20480
 210   pidfile  /tmp/haproxy.pid
 211   user  haproxy
 212
 213 defaults
 214   log  global
 215   maxconn  4096
 216   mode  tcp
 217   retries  3
 218   timeout  http-request 10s
 219   timeout  queue 1m
 220   timeout  connect 10s
 221   timeout  client 1m
 222   timeout  server 1m
 223   timeout  check 10s
 224
 225 listen opendaylight
 226   bind ${ha_proxy_ip}:8080
 227   balance source
 228 EOF
 229
 230 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 231 do
 232 odlip=ODL_SYSTEM_${i}_IP
 233 cat >> ${WORKSPACE}/haproxy.cfg << EOF
 234   server controller-$i ${!odlip}:8080 check fall 5 inter 2000 rise 2
 235 EOF
 236 done
 237
 238 cat > ${WORKSPACE}/deploy_ha_proxy.sh<< EOF
 239 sudo chown haproxy:haproxy /tmp/haproxy.cfg
 240 sudo sed -i 's/\\/etc\\/haproxy\\/haproxy.cfg/\\/tmp\\/haproxy.cfg/g' /usr/lib/systemd/system/haproxy.service
 241 sudo /usr/sbin/semanage permissive -a haproxy_t
 242 sudo systemctl restart haproxy
 243 sleep 3
 244 sudo netstat -tunpl
 245 sudo systemctl status haproxy
 246 true
 247 EOF
 248 scp ${WORKSPACE}/install_ha_proxy.sh ${ha_proxy_ip}:/tmp
 249 ssh ${ha_proxy_ip} "sudo bash /tmp/install_ha_proxy.sh"
 250 scp ${WORKSPACE}/haproxy.cfg ${ha_proxy_ip}:/tmp
 251 scp ${WORKSPACE}/deploy_ha_proxy.sh ${ha_proxy_ip}:/tmp
 252 ssh ${ha_proxy_ip} "sudo bash /tmp/deploy_ha_proxy.sh"
 253 }
 254
 255 function collect_logs_and_exit (){
 256 set +e  # We do not want to create red dot just because something went wrong while fetching logs.
 257 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 258 do
 259     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 260     echo "dumping first 500K bytes of karaf log..." > "odl${i}_karaf.log"
 261     ssh "${!CONTROLLERIP}" head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" >> "odl${i}_karaf.log"
 262     echo "dumping last 500K bytes of karaf log..." >> "odl${i}_karaf.log"
 263     ssh "${!CONTROLLERIP}" tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" >> "odl${i}_karaf.log"
 264     echo "killing karaf process..."
 265     ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
 266 done
 267 sleep 5
 268 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 269 do
 270     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 271     ssh "${!CONTROLLERIP}" xz -9ekvv "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
 272     scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.xz" "odl${i}_karaf.log.xz"
 273 done
 274
 275 ssh ${OPENSTACK_CONTROL_NODE_IP} "xz -9ekvv /opt/stack/devstack/nohup.out"
 276 scp ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/nohup.out.xz "openstack_control_stack.log.xz"
 277 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 278 do
 279     OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 280     scp "${!OSIP}:/opt/stack/devstack/nohup.out" "openstack_compute_stack_${i}.log"
 281 done
 282 }
 283
 284 cat > ${WORKSPACE}/get_devstack.sh << EOF
 285 sudo systemctl stop firewalld
 286 sudo yum install bridge-utils -y
 287 sudo systemctl stop  NetworkManager
 288 #Disable NetworkManager and kill dhclient and dnsmasq
 289 sudo systemctl stop NetworkManager
 290 sudo killall dhclient
 291 sudo killall dnsmasq
 292 #Workaround for mysql failure
 293 echo "127.0.0.1    localhost \${HOSTNAME}" > /tmp/hosts
 294 echo "::1   localhost  \${HOSTNAME}" >> /tmp/hosts
 295 sudo mv /tmp/hosts /etc/hosts
 296 sudo /usr/sbin/brctl addbr br100
 297 sudo mkdir /opt/stack
 298 sudo chmod 777 /opt/stack
 299 cd /opt/stack
 300 git clone https://git.openstack.org/openstack-dev/devstack
 301 cd devstack
 302 git checkout $OPENSTACK_BRANCH
 303 EOF
 304
 305 echo "Create HAProxy if needed"
 306 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 307  echo "Need to configure HAProxy"
 308  configure_haproxy_for_neutron_requests
 309 fi
 310
 311 os_node_list=()
 312 echo "Stack the Control Node"
 313 scp ${WORKSPACE}/get_devstack.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp
 314 ssh ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/get_devstack.sh"
 315 create_control_node_local_conf
 316 scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf
 317 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 318 ssh ${OPENSTACK_CONTROL_NODE_IP} "ps -ef | grep stack.sh"
 319 ssh ${OPENSTACK_CONTROL_NODE_IP} "ls -lrt /opt/stack/devstack/nohup.out"
 320 os_node_list+=(${OPENSTACK_CONTROL_NODE_IP})
 321
 322
 323 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 324 do
 325     COMPUTEIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 326     scp ${WORKSPACE}/get_devstack.sh  ${!COMPUTEIP}:/tmp
 327     ssh ${!COMPUTEIP} "bash /tmp/get_devstack.sh"
 328     create_compute_node_local_conf ${!COMPUTEIP}
 329     scp ${WORKSPACE}/local.conf_compute_${!COMPUTEIP} ${!COMPUTEIP}:/opt/stack/devstack/local.conf
 330     ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 331     ssh ${!COMPUTEIP} "ps -ef | grep stack.sh"
 332     os_node_list+=(${!COMPUTEIP})
 333 done
 334
 335 cat > ${WORKSPACE}/check_stacking.sh << EOF
 336 > /tmp/stack_progress
 337 ps -ef | grep "stack.sh" | grep -v grep
 338 ret=\$?
 339 if [ \${ret} -eq 1 ]; then
 340   grep "This is your host IP address:" /opt/stack/devstack/nohup.out
 341   if [ \$? -eq 0 ]; then
 342      echo "Stacking Complete" > /tmp/stack_progress
 343   else
 344      echo "Stacking Failed" > /tmp/stack_progress
 345   fi
 346 elif [ \${ret} -eq 0 ]; then
 347   echo "Still Stacking" > /tmp/stack_progress
 348 fi
 349 EOF
 350
 351 #the checking is repeated for an hour
 352 iteration=0
 353 in_progress=1
 354 while [ ${in_progress} -eq 1 ]; do
 355 iterator=$(($iterator + 1))
 356 for index in ${!os_node_list[@]}
 357 do
 358 echo "Check the status of stacking in ${os_node_list[index]}"
 359 scp ${WORKSPACE}/check_stacking.sh  ${os_node_list[index]}:/tmp
 360 ssh ${os_node_list[index]} "bash /tmp/check_stacking.sh"
 361 scp ${os_node_list[index]}:/tmp/stack_progress .
 362 #debug
 363 cat stack_progress
 364 stacking_status=`cat stack_progress`
 365 if [ "$stacking_status" == "Still Stacking" ]; then
 366   continue
 367 elif [ "$stacking_status" == "Stacking Failed" ]; then
 368   collect_logs_and_exit
 369   exit 1
 370 elif [ "$stacking_status" == "Stacking Complete" ]; then
 371   unset os_node_list[index]
 372   if  [ ${#os_node_list[@]} -eq 0 ]; then
 373      in_progress=0
 374   fi
 375 fi
 376 done
 377  echo "sleep for a minute before the next check"
 378  sleep 60
 379  if [ ${iteration} -eq 60 ]; then
 380   collect_logs_and_exit
 381   exit 1
 382  fi
 383 done
 384
 385 #Need to disable firewalld and iptables in control node
 386 echo "Stop Firewall in Control Node for compute nodes to be able to reach the ports and add to hypervisor-list"
 387 ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo systemctl stop firewalld; sudo systemctl stop iptables"
 388 echo "sleep for a minute and print hypervisor-list"
 389 sleep 60
 390 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; source openrc admin admin; nova hypervisor-list"
 391
 392 #Need to disable firewalld and iptables in compute 1 node
 393 echo "Stop Firewall in Compute 1 Node and add to hypervisor-list"
 394 ssh ${OPENSTACK_COMPUTE_NODE_1_IP} "sudo systemctl stop firewalld; sudo systemctl stop iptables"
 395 #Need to disable firewalld and iptables in compute 2 node
 396 echo "Stop Firewall in Compute 2 Node and add to hypervisor-list"
 397 ssh ${OPENSTACK_COMPUTE_NODE_2_IP} "sudo systemctl stop firewalld; sudo systemctl stop iptables"
 398
 399 echo "Locating test plan to use..."
 400 testplan_filepath="${WORKSPACE}/test/csit/testplans/${STREAMTESTPLAN}"
 401 if [ ! -f "${testplan_filepath}" ]; then
 402     testplan_filepath="${WORKSPACE}/test/csit/testplans/${TESTPLAN}"
 403 fi
 404
 405 echo "Changing the testplan path..."
 406 cat "${testplan_filepath}" | sed "s:integration:${WORKSPACE}:" > testplan.txt
 407 cat testplan.txt
 408
 409 SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
 410
 411 echo "Starting Robot test suites ${SUITES} ..."
 412 pybot -N ${TESTPLAN} -c critical -e exclude -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp -v BUNDLE_URL:${ACTUALBUNDLEURL} \
 413 -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} -v JDKVERSION:${JDKVERSION} -v ODL_STREAM:${DISTROSTREAM} \
 414 -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} -v ODL_SYSTEM_1_IP:${ODL_SYSTEM_1_IP} -v ODL_SYSTEM_2_IP:${ODL_SYSTEM_2_IP} \
 415 -v ODL_SYSTEM_3_IP:${ODL_SYSTEM_3_IP} -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} -v CONTROLLER_USER:${USER} -v OS_USER:${USER} \
 416 -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \
 417 -v OS_COMPUTE_1_IP:${OPENSTACK_COMPUTE_NODE_1_IP} -v OS_COMPUTE_2_IP:${OPENSTACK_COMPUTE_NODE_2_IP} \
 418 -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack -v USER_HOME:${HOME} ${TESTOPTIONS} ${SUITES} || true
 419
 420 echo "Tests Executed"
 421 collect_logs_and_exit
 422
 423 true  # perhaps Jenkins is testing last exit code
 424 # vim: ts=4 sw=4 sts=4 et ft=sh :