jjb/integration/include-raw-integration-deploy-openstack-run-test.sh

   1 #@IgnoreInspection BashAddShebang
   2 # Activate robotframework virtualenv
   3 # ${ROBOT_VENV} comes from the include-raw-integration-install-robotframework.sh
   4 # script.
   5 source ${ROBOT_VENV}/bin/activate
   6
   7 echo "#################################################"
   8 echo "##         Deploy Openstack 3-node             ##"
   9 echo "#################################################"
  10
  11 function create_control_node_local_conf {
  12 local_conf_file_name=${WORKSPACE}/local.conf_control
  13 cat > ${local_conf_file_name} << EOF
  14 [[local|localrc]]
  15 LOGFILE=stack.sh.log
  16 SCREEN_LOGDIR=/opt/stack/data/log
  17 LOG_COLOR=False
  18 RECLONE=yes
  19
  20 disable_service swift
  21 disable_service cinder
  22 disable_service n-net
  23 disable_service q-vpn
  24 enable_service q-svc
  25 enable_service q-dhcp
  26 enable_service q-meta
  27 enable_service tempest
  28 enable_service n-novnc
  29 enable_service n-cauth
  30
  31 HOST_IP=$OPENSTACK_CONTROL_NODE_IP
  32 SERVICE_HOST=\$HOST_IP
  33
  34 NEUTRON_CREATE_INITIAL_NETWORKS=False
  35 Q_PLUGIN=ml2
  36 Q_ML2_TENANT_NETWORK_TYPE=vxlan
  37
  38 ENABLE_TENANT_TUNNELS=True
  39
  40
  41 MYSQL_HOST=\$SERVICE_HOST
  42 RABBIT_HOST=\$SERVICE_HOST
  43 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
  44 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
  45 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
  46
  47 MYSQL_PASSWORD=mysql
  48 RABBIT_PASSWORD=rabbit
  49 SERVICE_TOKEN=service
  50 SERVICE_PASSWORD=admin
  51 ADMIN_PASSWORD=admin
  52
  53 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
  54
  55 ODL_PORT=8080
  56 ODL_MODE=externalodl
  57
  58 EOF
  59
  60 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
  61 odl_list=${ODL_SYSTEM_1_IP}
  62 for i in `seq 2 ${NUM_ODL_SYSTEM}`
  63 do
  64 odlip=ODL_SYSTEM_${i}_IP
  65 odl_list=${odl_list},${!odlip}
  66 done
  67 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
  68 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
  69 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
  70 odl_mgr_ip=${!odlmgrip}
  71 else
  72 odl_mgr_ip=${ODL_SYSTEM_1_IP}
  73 fi
  74 cat >> ${local_conf_file_name} << EOF
  75 ODL_OVS_MANAGERS=${odl_list}
  76 ODL_MGR_IP=${odl_mgr_ip}
  77 EOF
  78 else
  79 cat >> ${local_conf_file_name} << EOF
  80 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
  81 EOF
  82 fi
  83
  84 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
  85 cat >> ${local_conf_file_name} << EOF
  86
  87 ODL_PROVIDER_MAPPINGS=br-ex:br100
  88
  89 disable_service q-l3
  90 Q_L3_ENABLED=True
  91 ODL_L3=True
  92 PUBLIC_INTERFACE=br100
  93 [[post-config|\$NEUTRON_CONF]]
  94 [DEFAULT]
  95 service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
  96
  97 EOF
  98 fi
  99 cat >> ${local_conf_file_name} << EOF
 100 [[post-config|/etc/neutron/plugins/ml2/ml2_conf.ini]]
 101 [agent]
 102 minimize_polling=True
 103
 104 [[post-config|/etc/neutron/dhcp_agent.ini]]
 105 [DEFAULT]
 106 force_metadata = True
 107 enable_isolated_metadata = True
 108
 109 [[post-config|/etc/nova/nova.conf]]
 110 [DEFAULT]
 111 force_config_drive = False
 112
 113 EOF
 114
 115 echo "local.conf Created...."
 116 cat ${local_conf_file_name}
 117 }
 118
 119 function create_compute_node_local_conf {
 120 HOSTIP=$1
 121 local_conf_file_name=${WORKSPACE}/local.conf_compute_${HOSTIP}
 122 cat > ${local_conf_file_name} << EOF
 123 [[local|localrc]]
 124 LOGFILE=stack.sh.log
 125 LOG_COLOR=False
 126 SCREEN_LOGDIR=/opt/stack/data/log
 127 RECLONE=yes
 128
 129 NOVA_VNC_ENABLED=True
 130 MULTI_HOST=1
 131 ENABLED_SERVICES=n-cpu
 132
 133 HOST_IP=${HOSTIP}
 134 SERVICE_HOST=${OPENSTACK_CONTROL_NODE_IP}
 135
 136 Q_PLUGIN=ml2
 137 ENABLE_TENANT_TUNNELS=True
 138 Q_ML2_TENANT_NETWORK_TYPE=vxlan
 139
 140 Q_HOST=\$SERVICE_HOST
 141 MYSQL_HOST=\$SERVICE_HOST
 142 RABBIT_HOST=\$SERVICE_HOST
 143 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
 144 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
 145 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
 146
 147 MYSQL_PASSWORD=mysql
 148 RABBIT_PASSWORD=rabbit
 149 SERVICE_TOKEN=service
 150 SERVICE_PASSWORD=admin
 151 ADMIN_PASSWORD=admin
 152
 153 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
 154 ODL_MODE=compute
 155
 156 EOF
 157
 158 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
 159 odl_list=${ODL_SYSTEM_1_IP}
 160 for i in `seq 2 ${NUM_ODL_SYSTEM}`
 161 do
 162 odlip=ODL_SYSTEM_${i}_IP
 163 odl_list=${odl_list},${!odlip}
 164 done
 165 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 166 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 167 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 168 odl_mgr_ip=${!odlmgrip}
 169 else
 170 odl_mgr_ip=${ODL_SYSTEM_1_IP}
 171 fi
 172 cat >> ${local_conf_file_name} << EOF
 173 ODL_OVS_MANAGERS=${odl_list}
 174 ODL_MGR_IP=${odl_mgr_ip}
 175 EOF
 176 else
 177 cat >> ${local_conf_file_name} << EOF
 178 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
 179 EOF
 180 fi
 181
 182 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
 183 cat >> ${local_conf_file_name} << EOF
 184 # Uncomment lines below if odl-compute is to be used for l3 forwarding
 185 Q_L3_ENABLED=True
 186 ODL_L3=True
 187 PUBLIC_INTERFACE=br100
 188 EOF
 189 fi
 190 echo "local.conf Created...."
 191 cat ${local_conf_file_name}
 192 }
 193
 194 function configure_haproxy_for_neutron_requests () {
 195 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 196 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 197 ha_proxy_ip=${!odlmgrip}
 198
 199 cat > ${WORKSPACE}/install_ha_proxy.sh<< EOF
 200 sudo systemctl stop firewalld
 201 sudo yum -y install policycoreutils-python haproxy
 202 EOF
 203
 204 cat > ${WORKSPACE}/haproxy.cfg << EOF
 205 global
 206   daemon
 207   group  haproxy
 208   log  /dev/log local0
 209   maxconn  20480
 210   pidfile  /tmp/haproxy.pid
 211   user  haproxy
 212
 213 defaults
 214   log  global
 215   maxconn  4096
 216   mode  tcp
 217   retries  3
 218   timeout  http-request 10s
 219   timeout  queue 1m
 220   timeout  connect 10s
 221   timeout  client 1m
 222   timeout  server 1m
 223   timeout  check 10s
 224
 225 listen opendaylight
 226   bind ${ha_proxy_ip}:8080
 227   balance source
 228 EOF
 229
 230 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 231 do
 232 odlip=ODL_SYSTEM_${i}_IP
 233 cat >> ${WORKSPACE}/haproxy.cfg << EOF
 234   server controller-$i ${!odlip}:8080 check fall 5 inter 2000 rise 2
 235 EOF
 236 done
 237
 238 cat > ${WORKSPACE}/deploy_ha_proxy.sh<< EOF
 239 sudo chown haproxy:haproxy /tmp/haproxy.cfg
 240 sudo sed -i 's/\\/etc\\/haproxy\\/haproxy.cfg/\\/tmp\\/haproxy.cfg/g' /usr/lib/systemd/system/haproxy.service
 241 sudo /usr/sbin/semanage permissive -a haproxy_t
 242 sudo systemctl restart haproxy
 243 sleep 3
 244 sudo netstat -tunpl
 245 sudo systemctl status haproxy
 246 true
 247 EOF
 248 scp ${WORKSPACE}/install_ha_proxy.sh ${ha_proxy_ip}:/tmp
 249 ssh ${ha_proxy_ip} "sudo bash /tmp/install_ha_proxy.sh"
 250 scp ${WORKSPACE}/haproxy.cfg ${ha_proxy_ip}:/tmp
 251 scp ${WORKSPACE}/deploy_ha_proxy.sh ${ha_proxy_ip}:/tmp
 252 ssh ${ha_proxy_ip} "sudo bash /tmp/deploy_ha_proxy.sh"
 253 }
 254
 255 function collect_logs_and_exit (){
 256 set +e  # We do not want to create red dot just because something went wrong while fetching logs.
 257 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 258 do
 259     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 260     echo "dumping first 500K bytes of karaf log..." > "odl${i}_karaf.log"
 261     ssh "${!CONTROLLERIP}" head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" >> "odl${i}_karaf.log"
 262     echo "dumping last 500K bytes of karaf log..." >> "odl${i}_karaf.log"
 263     ssh "${!CONTROLLERIP}" tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" >> "odl${i}_karaf.log"
 264     echo "killing karaf process..."
 265     ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
 266 done
 267 sleep 5
 268 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 269 do
 270     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 271     ssh "${!CONTROLLERIP}" xz -9ekvv "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
 272     scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.xz" "odl${i}_karaf.log.xz"
 273 done
 274
 275 ssh ${OPENSTACK_CONTROL_NODE_IP} "xz -9ekvv /opt/stack/devstack/nohup.out"
 276 scp ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/nohup.out.xz "openstack_control_stack.log.xz"
 277 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 278 do
 279     OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 280     scp "${!OSIP}:/opt/stack/devstack/nohup.out" "openstack_compute_stack_${i}.log"
 281 done
 282 }
 283
 284 cat > ${WORKSPACE}/get_devstack.sh << EOF
 285 sudo systemctl stop firewalld
 286 sudo yum install bridge-utils -y
 287 #Workaround for mysql failure
 288 echo "127.0.0.1    localhost \${HOSTNAME}" > /tmp/hosts
 289 echo "::1   localhost  \${HOSTNAME}" >> /tmp/hosts
 290 sudo mv /tmp/hosts /etc/hosts
 291 sudo /usr/sbin/brctl addbr br100
 292 sudo mkdir /opt/stack
 293 sudo chmod 777 /opt/stack
 294 cd /opt/stack
 295 git clone https://git.openstack.org/openstack-dev/devstack
 296 cd devstack
 297 git checkout $OPENSTACK_BRANCH
 298 EOF
 299
 300 echo "Create HAProxy if needed"
 301 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 302  echo "Need to configure HAProxy"
 303  configure_haproxy_for_neutron_requests
 304 fi
 305
 306 os_node_list=()
 307 echo "Stack the Control Node"
 308 scp ${WORKSPACE}/get_devstack.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp
 309 ssh ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/get_devstack.sh"
 310 create_control_node_local_conf
 311 scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf
 312 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 313 ssh ${OPENSTACK_CONTROL_NODE_IP} "ps -ef | grep stack.sh"
 314 ssh ${OPENSTACK_CONTROL_NODE_IP} "ls -lrt /opt/stack/devstack/nohup.out"
 315 os_node_list+=(${OPENSTACK_CONTROL_NODE_IP})
 316
 317
 318 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 319 do
 320     COMPUTEIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 321     scp ${WORKSPACE}/get_devstack.sh  ${!COMPUTEIP}:/tmp
 322     ssh ${!COMPUTEIP} "bash /tmp/get_devstack.sh"
 323     create_compute_node_local_conf ${!COMPUTEIP}
 324     scp ${WORKSPACE}/local.conf_compute_${!COMPUTEIP} ${!COMPUTEIP}:/opt/stack/devstack/local.conf
 325     ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 326     ssh ${!COMPUTEIP} "ps -ef | grep stack.sh"
 327     os_node_list+=(${!COMPUTEIP})
 328 done
 329
 330 cat > ${WORKSPACE}/check_stacking.sh << EOF
 331 > /tmp/stack_progress
 332 ps -ef | grep "stack.sh" | grep -v grep
 333 ret=\$?
 334 if [ \${ret} -eq 1 ]; then
 335   grep "This is your host IP address:" /opt/stack/devstack/nohup.out
 336   if [ \$? -eq 0 ]; then
 337      echo "Stacking Complete" > /tmp/stack_progress
 338   else
 339      echo "Stacking Failed" > /tmp/stack_progress
 340   fi
 341 elif [ \${ret} -eq 0 ]; then
 342   echo "Still Stacking" > /tmp/stack_progress
 343 fi
 344 EOF
 345
 346 #the checking is repeated for an hour
 347 iteration=0
 348 in_progress=1
 349 while [ ${in_progress} -eq 1 ]; do
 350 iterator=$(($iterator + 1))
 351 for index in ${!os_node_list[@]}
 352 do
 353 echo "Check the status of stacking in ${os_node_list[index]}"
 354 scp ${WORKSPACE}/check_stacking.sh  ${os_node_list[index]}:/tmp
 355 ssh ${os_node_list[index]} "bash /tmp/check_stacking.sh"
 356 scp ${os_node_list[index]}:/tmp/stack_progress .
 357 #debug
 358 cat stack_progress
 359 stacking_status=`cat stack_progress`
 360 if [ "$stacking_status" == "Still Stacking" ]; then
 361   continue
 362 elif [ "$stacking_status" == "Stacking Failed" ]; then
 363   collect_logs_and_exit
 364   exit 1
 365 elif [ "$stacking_status" == "Stacking Complete" ]; then
 366   unset os_node_list[index]
 367   if  [ ${#os_node_list[@]} -eq 0 ]; then
 368      in_progress=0
 369   fi
 370 fi
 371 done
 372  echo "sleep for a minute before the next check"
 373  sleep 60
 374  if [ ${iteration} -eq 60 ]; then
 375   collect_logs_and_exit
 376   exit 1
 377  fi
 378 done
 379
 380 echo "Locating test plan to use..."
 381 testplan_filepath="${WORKSPACE}/test/csit/testplans/${STREAMTESTPLAN}"
 382 if [ ! -f "${testplan_filepath}" ]; then
 383     testplan_filepath="${WORKSPACE}/test/csit/testplans/${TESTPLAN}"
 384 fi
 385
 386 echo "Changing the testplan path..."
 387 cat "${testplan_filepath}" | sed "s:integration:${WORKSPACE}:" > testplan.txt
 388 cat testplan.txt
 389
 390 SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
 391
 392 echo "Starting Robot test suites ${SUITES} ..."
 393 pybot -N ${TESTPLAN} -c critical -e exclude -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp -v BUNDLE_URL:${ACTUALBUNDLEURL} \
 394 -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} -v JDKVERSION:${JDKVERSION} -v ODL_STREAM:${DISTROSTREAM} \
 395 -v CONTROLLER:${ODL_SYSTEM_IP} -v CONTROLLER1:${ODL_SYSTEM_2_IP} -v CONTROLLER2:${ODL_SYSTEM_3_IP} -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} \
 396 -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} -v CONTROLLER_USER:${USER} -v OS_USER:${USER}  \
 397 -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \
 398 -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack -v USER_HOME:${HOME} ${TESTOPTIONS} ${SUITES} || true
 399
 400 echo "Tests Executed"
 401 collect_logs_and_exit
 402
 403 true  # perhaps Jenkins is testing last exit code
 404 # vim: ts=4 sw=4 sts=4 et ft=sh :