X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Fopendaylight-infra-stack.sh;h=381f9bec3f6a981992609e3d8d8f9359290a72b9;hb=53d9e249a24624cdf89c2c3c67128de34110dfe1;hp=45eaae9f4c5b13ea623af4584aed9b6f568ef597;hpb=5d4341b46a6d1875befc576ebc63d385ebd48181;p=releng%2Fbuilder.git diff --git a/jjb/opendaylight-infra-stack.sh b/jjb/opendaylight-infra-stack.sh index 45eaae9f4..381f9bec3 100644 --- a/jjb/opendaylight-infra-stack.sh +++ b/jjb/opendaylight-infra-stack.sh @@ -1,8 +1,18 @@ #!/bin/bash -virtualenv "$WORKSPACE/.venv-openstack" -# shellcheck disable=SC1090 -source "$WORKSPACE/.venv-openstack/bin/activate" -pip install --upgrade pip +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +virtualenv "/tmp/v/openstack" +# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091 +source "/tmp/v/openstack/bin/activate" +pip install --upgrade "pip<10.0.0" setuptools pip install --upgrade python-openstackclient python-heatclient pip freeze @@ -11,8 +21,8 @@ cd /builder/openstack-hot || exit 1 JOB_SUM=$(echo "$JOB_NAME" | sum | awk '{{ print $1 }}') VM_NAME="$JOB_SUM-$BUILD_NUMBER" -OS_TIMEOUT=10 # Minutes to wait for OpenStack VM to come online -STACK_RETRIES=3 # Number of times to retry creating a stack before fully giving up +OS_TIMEOUT=15 # Minutes to wait for OpenStack VM to come online +STACK_RETRIES=2 # Number of times to retry creating a stack before fully giving up STACK_SUCCESSFUL=false # seq X refers to waiting for X minutes for OpenStack to return # a status that is not CREATE_IN_PROGRESS before giving up. @@ -25,7 +35,7 @@ for try in $(seq $STACK_RETRIES); do echo "$try: Waiting for $OS_TIMEOUT minutes to create $STACK_NAME." for i in $(seq $OS_TIMEOUT); do sleep 60 - OS_STATUS=$(openstack stack show -f json -c stack_status "$STACK_NAME" | jq -r '.stack_status') + OS_STATUS=$(openstack stack show -f value -c stack_status "$STACK_NAME") echo "$i: $OS_STATUS" case "$OS_STATUS" in @@ -35,39 +45,42 @@ for try in $(seq $STACK_RETRIES); do break ;; CREATE_FAILED) - echo "ERROR: Failed to initialize infrastructure. Deleting stack and possibly retrying to create..." + reason=$(openstack stack show "$STACK_NAME" -f value -c stack_status_reason) + echo "ERROR: Failed to initialize infrastructure. Reason: $reason" + openstack stack resource list -n 25 "$STACK_NAME" + + echo "Deleting stack and possibly retrying to create..." openstack stack delete --yes "$STACK_NAME" - openstack stack show "$STACK_NAME" + # after stack delete, poll for 10m to know when stack is fully removed # the logic here is that when "stack show $STACK_NAME" does not contain $STACK_NAME # we assume it's successfully deleted and we can break to retry for j in $(seq 20); do - sleep 30; - STACK_SHOW=$(openstack stack show "$STACK_NAME") - echo "$j: $STACK_SHOW" - if [[ $STACK_SHOW == *"DELETE_FAILED"* ]]; then - echo "stack delete failed. trying to stack abandon now" - # stack abandon does not work on RS, therefore requires acquiring a token - # and using http delete method to abondon DELETE_FAILED stacks - # Todo: remove the change once RS fixes the issue upstream + sleep 30 + delete_status=$(openstack stack show "$STACK_NAME" -f value -c stack_status) + echo "$j: $delete_status" + if [[ $delete_status == "DELETE_FAILED" ]]; then + reason=$(openstack stack show "$STACK_NAME" -f value -c stack_status_reason) + echo "ERROR: Failed to delete $STACK_NAME. Reason: $reason" + + # Abandon is not supported in Vexxhost so let's keep trying to + # delete for now... + # echo "Stack delete failed, trying to stack abandon now." # openstack stack abandon "$STACK_NAME" - STACK_ID=$(openstack stack show -f json -c "id" "$STACK_NAME" | jq -r '."id"') - TOKEN=$(openstack token issue -f json -c id | jq -r '.id') - curl -si -X DELETE -H "Content-Type: application/json" -H "Accept: application/json"\ - -H "x-auth-token: $TOKEN"\ - "https://dfw.orchestration.api.rackspacecloud.com/v1/904885/stacks/$STACK_NAME/$STACK_ID/abandon" - STACK_SHOW=$(openstack stack show "$STACK_NAME") - echo "$STACK_SHOW" + echo "Deleting failed stack: $STACK_NAME" + openstack stack delete --yes "$STACK_NAME" fi - if [[ $STACK_SHOW != *"$STACK_NAME"* ]]; then - echo "stack show on $STACK_NAME came back empty. Assuming successful delete" + + if ! openstack stack show "$STACK_NAME" -f value -c stack_status; then + echo "Stack show on $STACK_NAME came back empty. Assuming successful delete" break fi done - # if we still see $STACK_NAME in $STACK_SHOW it means the delete hasn't fully + + # If we still see $STACK_NAME in `openstack stack show` it means the delete hasn't fully # worked and we can exit forcefully - if [[ $STACK_SHOW == *"$STACK_NAME"* ]]; then - echo "stack $STACK_NAME still in stack show output after polling. Quitting!" + if openstack stack show "$STACK_NAME" -f value -c stack_status; then + echo "Stack $STACK_NAME still in cloud output after polling. Quitting!" exit 1 fi break @@ -78,7 +91,10 @@ for try in $(seq $STACK_RETRIES); do ;; *) echo "Unexpected status: $OS_STATUS" - exit 1 + # DO NOT exit on unexpected status. Rackspace sometimes returns unexpected status + # before returning an expected status. Just print the message and loop until we have + # a confirmed state or timeout. + # exit 1 ;; esac done @@ -88,7 +104,11 @@ for try in $(seq $STACK_RETRIES); do done # capture stack info in console logs -openstack stack show "$STACK_NAME" +echo "------------------------------------" +echo "Stack details" +echo "------------------------------------" +openstack stack show "$STACK_NAME" -f yaml +echo "------------------------------------" if ! $STACK_SUCCESSFUL; then exit 1