jjb/integration/include-raw-integration-deploy-openstack-run-test.sh

   1 #@IgnoreInspection BashAddShebang
   2 # Activate robotframework virtualenv
   3 # ${ROBOT_VENV} comes from the include-raw-integration-install-robotframework.sh
   4 # script.
   5 source ${ROBOT_VENV}/bin/activate
   6
   7 echo "#################################################"
   8 echo "##         Deploy Openstack 3-node             ##"
   9 echo "#################################################"
  10
  11 function create_control_node_local_conf {
  12 local_conf_file_name=${WORKSPACE}/local.conf_control
  13 cat > ${local_conf_file_name} << EOF
  14 [[local|localrc]]
  15 LOGFILE=stack.sh.log
  16 SCREEN_LOGDIR=/opt/stack/data/log
  17 LOG_COLOR=False
  18 RECLONE=yes
  19
  20 disable_service swift
  21 disable_service cinder
  22 disable_service n-net
  23 disable_service q-vpn
  24 enable_service q-svc
  25 enable_service q-dhcp
  26 enable_service q-meta
  27 enable_service tempest
  28 enable_service n-novnc
  29 enable_service n-cauth
  30
  31 HOST_IP=$OPENSTACK_CONTROL_NODE_IP
  32 SERVICE_HOST=\$HOST_IP
  33
  34 NEUTRON_CREATE_INITIAL_NETWORKS=False
  35 Q_PLUGIN=ml2
  36 Q_ML2_TENANT_NETWORK_TYPE=vxlan
  37
  38 ENABLE_TENANT_TUNNELS=True
  39
  40
  41 MYSQL_HOST=\$SERVICE_HOST
  42 RABBIT_HOST=\$SERVICE_HOST
  43 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
  44 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
  45 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
  46
  47 MYSQL_PASSWORD=mysql
  48 RABBIT_PASSWORD=rabbit
  49 SERVICE_TOKEN=service
  50 SERVICE_PASSWORD=admin
  51 ADMIN_PASSWORD=admin
  52
  53 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
  54
  55 ODL_PORT=8080
  56 ODL_MODE=externalodl
  57
  58 EOF
  59
  60 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
  61 odl_list=${ODL_SYSTEM_1_IP}
  62 for i in `seq 2 ${NUM_ODL_SYSTEM}`
  63 do
  64 odlip=ODL_SYSTEM_${i}_IP
  65 odl_list=${odl_list},${!odlip}
  66 done
  67 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
  68 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
  69 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
  70 odl_mgr_ip=${!odlmgrip}
  71 else
  72 odl_mgr_ip=${ODL_SYSTEM_1_IP}
  73 fi
  74 cat >> ${local_conf_file_name} << EOF
  75 ODL_OVS_MANAGERS=${odl_list}
  76 ODL_MGR_IP=${odl_mgr_ip}
  77 EOF
  78 else
  79 cat >> ${local_conf_file_name} << EOF
  80 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
  81 EOF
  82 fi
  83
  84 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
  85 cat >> ${local_conf_file_name} << EOF
  86
  87 ODL_PROVIDER_MAPPINGS=br-ex:br100
  88
  89 disable_service q-l3
  90 Q_L3_ENABLED=True
  91 ODL_L3=True
  92 PUBLIC_INTERFACE=br100
  93 [[post-config|\$NEUTRON_CONF]]
  94 [DEFAULT]
  95 service_plugins = networking_odl.l3.l3_odl.OpenDaylightL3RouterPlugin
  96
  97 EOF
  98 fi
  99 cat >> ${local_conf_file_name} << EOF
 100 [[post-config|/etc/neutron/plugins/ml2/ml2_conf.ini]]
 101 [agent]
 102 minimize_polling=True
 103 EOF
 104
 105 echo "local.conf Created...."
 106 cat ${local_conf_file_name}
 107 }
 108
 109 function create_compute_node_local_conf {
 110 HOSTIP=$1
 111 local_conf_file_name=${WORKSPACE}/local.conf_compute_${HOSTIP}
 112 cat > ${local_conf_file_name} << EOF
 113 [[local|localrc]]
 114 LOGFILE=stack.sh.log
 115 LOG_COLOR=False
 116 SCREEN_LOGDIR=/opt/stack/data/log
 117 RECLONE=yes
 118
 119 NOVA_VNC_ENABLED=True
 120 MULTI_HOST=1
 121 ENABLED_SERVICES=n-cpu
 122
 123 HOST_IP=${HOSTIP}
 124 SERVICE_HOST=${OPENSTACK_CONTROL_NODE_IP}
 125
 126 Q_PLUGIN=ml2
 127 ENABLE_TENANT_TUNNELS=True
 128 Q_ML2_TENANT_NETWORK_TYPE=vxlan
 129
 130 Q_HOST=\$SERVICE_HOST
 131 MYSQL_HOST=\$SERVICE_HOST
 132 RABBIT_HOST=\$SERVICE_HOST
 133 GLANCE_HOSTPORT=\$SERVICE_HOST:9292
 134 KEYSTONE_AUTH_HOST=\$SERVICE_HOST
 135 KEYSTONE_SERVICE_HOST=\$SERVICE_HOST
 136
 137 MYSQL_PASSWORD=mysql
 138 RABBIT_PASSWORD=rabbit
 139 SERVICE_TOKEN=service
 140 SERVICE_PASSWORD=admin
 141 ADMIN_PASSWORD=admin
 142
 143 enable_plugin networking-odl ${ODL_ML2_DRIVER_REPO} ${ODL_ML2_BRANCH}
 144 ODL_MODE=compute
 145
 146 EOF
 147
 148 if [ "${NUM_ODL_SYSTEM}" -gt 1 ]; then
 149 odl_list=${ODL_SYSTEM_1_IP}
 150 for i in `seq 2 ${NUM_ODL_SYSTEM}`
 151 do
 152 odlip=ODL_SYSTEM_${i}_IP
 153 odl_list=${odl_list},${!odlip}
 154 done
 155 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 156 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 157 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 158 odl_mgr_ip=${!odlmgrip}
 159 else
 160 odl_mgr_ip=${ODL_SYSTEM_1_IP}
 161 fi
 162 cat >> ${local_conf_file_name} << EOF
 163 ODL_OVS_MANAGERS=${odl_list}
 164 ODL_MGR_IP=${odl_mgr_ip}
 165 EOF
 166 else
 167 cat >> ${local_conf_file_name} << EOF
 168 ODL_MGR_IP=${ODL_SYSTEM_1_IP}
 169 EOF
 170 fi
 171
 172 if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
 173 cat >> ${local_conf_file_name} << EOF
 174 # Uncomment lines below if odl-compute is to be used for l3 forwarding
 175 Q_L3_ENABLED=True
 176 ODL_L3=True
 177 PUBLIC_INTERFACE=br100
 178 EOF
 179 fi
 180 echo "local.conf Created...."
 181 cat ${local_conf_file_name}
 182 }
 183
 184 function configure_haproxy_for_neutron_requests () {
 185 HA_PROXY_INDEX=${NUM_OPENSTACK_SYSTEM}
 186 odlmgrip=OPENSTACK_COMPUTE_NODE_${HA_PROXY_INDEX}_IP
 187 ha_proxy_ip=${!odlmgrip}
 188
 189 cat > ${WORKSPACE}/install_ha_proxy.sh<< EOF
 190 sudo systemctl stop firewalld
 191 sudo yum -y install policycoreutils-python haproxy
 192 EOF
 193
 194 cat > ${WORKSPACE}/haproxy.cfg << EOF
 195 global
 196   daemon
 197   group  haproxy
 198   log  /dev/log local0
 199   maxconn  20480
 200   pidfile  /tmp/haproxy.pid
 201   user  haproxy
 202
 203 defaults
 204   log  global
 205   maxconn  4096
 206   mode  tcp
 207   retries  3
 208   timeout  http-request 10s
 209   timeout  queue 1m
 210   timeout  connect 10s
 211   timeout  client 1m
 212   timeout  server 1m
 213   timeout  check 10s
 214
 215 listen opendaylight
 216   bind ${ha_proxy_ip}:8080
 217   balance source
 218 EOF
 219
 220 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 221 do
 222 odlip=ODL_SYSTEM_${i}_IP
 223 cat >> ${WORKSPACE}/haproxy.cfg << EOF
 224   server controller-$i ${!odlip}:8080 check fall 5 inter 2000 rise 2
 225 EOF
 226 done
 227
 228 cat > ${WORKSPACE}/deploy_ha_proxy.sh<< EOF
 229 sudo chown haproxy:haproxy /tmp/haproxy.cfg
 230 sudo sed -i 's/\\/etc\\/haproxy\\/haproxy.cfg/\\/tmp\\/haproxy.cfg/g' /usr/lib/systemd/system/haproxy.service
 231 sudo /usr/sbin/semanage permissive -a haproxy_t
 232 sudo systemctl restart haproxy
 233 sleep 3
 234 sudo netstat -tunpl
 235 sudo systemctl status haproxy
 236 true
 237 EOF
 238 scp ${WORKSPACE}/install_ha_proxy.sh ${ha_proxy_ip}:/tmp
 239 ssh ${ha_proxy_ip} "sudo bash /tmp/install_ha_proxy.sh"
 240 scp ${WORKSPACE}/haproxy.cfg ${ha_proxy_ip}:/tmp
 241 scp ${WORKSPACE}/deploy_ha_proxy.sh ${ha_proxy_ip}:/tmp
 242 ssh ${ha_proxy_ip} "sudo bash /tmp/deploy_ha_proxy.sh"
 243 }
 244
 245 function collect_logs_and_exit (){
 246 set +e  # We do not want to create red dot just because something went wrong while fetching logs.
 247 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 248 do
 249     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 250     echo "dumping first 500K bytes of karaf log..." > "odl${i}_karaf.log"
 251     ssh "${!CONTROLLERIP}" head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" >> "odl${i}_karaf.log"
 252     echo "dumping last 500K bytes of karaf log..." >> "odl${i}_karaf.log"
 253     ssh "${!CONTROLLERIP}" tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" >> "odl${i}_karaf.log"
 254     echo "killing karaf process..."
 255     ssh "${!CONTROLLERIP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
 256 done
 257 sleep 5
 258 for i in `seq 1 ${NUM_ODL_SYSTEM}`
 259 do
 260     CONTROLLERIP=ODL_SYSTEM_${i}_IP
 261     ssh "${!CONTROLLERIP}" xz -9ekvv "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
 262     scp "${!CONTROLLERIP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.xz" "odl${i}_karaf.log.xz"
 263 done
 264
 265 ssh ${OPENSTACK_CONTROL_NODE_IP} "xz -9ekvv /opt/stack/devstack/nohup.out"
 266 scp ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/nohup.out.xz "openstack_control_stack.log.xz"
 267 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 268 do
 269     OSIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 270     scp "${!OSIP}:/opt/stack/devstack/nohup.out" "openstack_compute_stack_${i}.log"
 271 done
 272 }
 273
 274 cat > ${WORKSPACE}/get_devstack.sh << EOF
 275 sudo systemctl stop firewalld
 276 sudo yum install bridge-utils -y
 277 #Workaround for mysql failure
 278 echo "127.0.0.1    localhost \${HOSTNAME}" > /tmp/hosts
 279 echo "::1   localhost  \${HOSTNAME}" >> /tmp/hosts
 280 sudo mv /tmp/hosts /etc/hosts
 281 sudo /usr/sbin/brctl addbr br100
 282 sudo mkdir /opt/stack
 283 sudo chmod 777 /opt/stack
 284 cd /opt/stack
 285 git clone https://git.openstack.org/openstack-dev/devstack
 286 cd devstack
 287 git checkout $OPENSTACK_BRANCH
 288 EOF
 289
 290 echo "Create HAProxy if needed"
 291 if [ "${ENABLE_HAPROXY_FOR_NEUTRON}" == "yes" ]; then
 292  echo "Need to configure HAProxy"
 293  configure_haproxy_for_neutron_requests
 294 fi
 295
 296 os_node_list=()
 297 echo "Stack the Control Node"
 298 scp ${WORKSPACE}/get_devstack.sh ${OPENSTACK_CONTROL_NODE_IP}:/tmp
 299 ssh ${OPENSTACK_CONTROL_NODE_IP} "bash /tmp/get_devstack.sh"
 300 create_control_node_local_conf
 301 scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf
 302 ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 303 ssh ${OPENSTACK_CONTROL_NODE_IP} "ps -ef | grep stack.sh"
 304 ssh ${OPENSTACK_CONTROL_NODE_IP} "ls -lrt /opt/stack/devstack/nohup.out"
 305 os_node_list+=(${OPENSTACK_CONTROL_NODE_IP})
 306
 307
 308 for i in `seq 1 $((NUM_OPENSTACK_SYSTEM - 1))`
 309 do
 310     COMPUTEIP=OPENSTACK_COMPUTE_NODE_${i}_IP
 311     scp ${WORKSPACE}/get_devstack.sh  ${!COMPUTEIP}:/tmp
 312     ssh ${!COMPUTEIP} "bash /tmp/get_devstack.sh"
 313     create_compute_node_local_conf ${!COMPUTEIP}
 314     scp ${WORKSPACE}/local.conf_compute_${!COMPUTEIP} ${!COMPUTEIP}:/opt/stack/devstack/local.conf
 315     ssh ${!COMPUTEIP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
 316     ssh ${!COMPUTEIP} "ps -ef | grep stack.sh"
 317     os_node_list+=(${!COMPUTEIP})
 318 done
 319
 320 cat > ${WORKSPACE}/check_stacking.sh << EOF
 321 > /tmp/stack_progress
 322 ps -ef | grep "stack.sh" | grep -v grep
 323 ret=\$?
 324 if [ \${ret} -eq 1 ]; then
 325   grep "This is your host IP address:" /opt/stack/devstack/nohup.out
 326   if [ \$? -eq 0 ]; then
 327      echo "Stacking Complete" > /tmp/stack_progress
 328   else
 329      echo "Stacking Failed" > /tmp/stack_progress
 330   fi
 331 elif [ \${ret} -eq 0 ]; then
 332   echo "Still Stacking" > /tmp/stack_progress
 333 fi
 334 EOF
 335
 336 #the checking is repeated for an hour
 337 iteration=0
 338 in_progress=1
 339 while [ ${in_progress} -eq 1 ]; do
 340 iterator=$(($iterator + 1))
 341 for index in ${!os_node_list[@]}
 342 do
 343 echo "Check the status of stacking in ${os_node_list[index]}"
 344 scp ${WORKSPACE}/check_stacking.sh  ${os_node_list[index]}:/tmp
 345 ssh ${os_node_list[index]} "bash /tmp/check_stacking.sh"
 346 scp ${os_node_list[index]}:/tmp/stack_progress .
 347 #debug
 348 cat stack_progress
 349 stacking_status=`cat stack_progress`
 350 if [ "$stacking_status" == "Still Stacking" ]; then
 351   continue
 352 elif [ "$stacking_status" == "Stacking Failed" ]; then
 353   collect_logs_and_exit
 354   exit 1
 355 elif [ "$stacking_status" == "Stacking Complete" ]; then
 356   unset os_node_list[index]
 357   if  [ ${#os_node_list[@]} -eq 0 ]; then
 358      in_progress=0
 359   fi
 360 fi
 361 done
 362  echo "sleep for a minute before the next check"
 363  sleep 60
 364  if [ ${iteration} -eq 60 ]; then
 365   collect_logs_and_exit
 366   exit 1
 367  fi
 368 done
 369
 370 echo "Locating test plan to use..."
 371 testplan_filepath="${WORKSPACE}/test/csit/testplans/${STREAMTESTPLAN}"
 372 if [ ! -f "${testplan_filepath}" ]; then
 373     testplan_filepath="${WORKSPACE}/test/csit/testplans/${TESTPLAN}"
 374 fi
 375
 376 echo "Changing the testplan path..."
 377 cat "${testplan_filepath}" | sed "s:integration:${WORKSPACE}:" > testplan.txt
 378 cat testplan.txt
 379
 380 SUITES=`egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' '`
 381
 382 echo "Starting Robot test suites ${SUITES} ..."
 383 pybot -N ${TESTPLAN} -c critical -e exclude -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp -v BUNDLE_URL:${ACTUALBUNDLEURL} \
 384 -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} -v JDKVERSION:${JDKVERSION} -v ODL_STREAM:${DISTROSTREAM} \
 385 -v CONTROLLER:${ODL_SYSTEM_IP} -v CONTROLLER1:${ODL_SYSTEM_2_IP} -v CONTROLLER2:${ODL_SYSTEM_3_IP} -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} \
 386 -v NUM_ODL_SYSTEM:${NUM_ODL_SYSTEM} -v CONTROLLER_USER:${USER} -v OS_USER:${USER}  \
 387 -v NUM_OS_SYSTEM:${NUM_OPENSTACK_SYSTEM} -v OS_CONTROL_NODE_IP:${OPENSTACK_CONTROL_NODE_IP} \
 388 -v DEVSTACK_DEPLOY_PATH:/opt/stack/devstack -v USER_HOME:${HOME} ${TESTOPTIONS} ${SUITES} || true
 389
 390 echo "Tests Executed"
 391 collect_logs_and_exit
 392
 393 true  # perhaps Jenkins is testing last exit code
 394 # vim: ts=4 sw=4 sts=4 et ft=sh :