Improve Heat implementation error handling

Using Heat has increased our ability to spawn 33% more robot builders
but we're now hitting some issues with Rackspace that we will need to
the error handling of our scripts for when Rackspace returns a status
such as Error: None, or CREATE_IN_PROGRESS due to stacks taking longer
than expected to spawn. Rackspace has told us we cannot delete stacks
that are in CREATE_IN_PROGRESS or DELETE_IN_PROGRESS states and that we
should exit our code carefully.

This patch makes the following changes:

* Wait and query stack create status every minute for 15 minutes
* Check status is CREATE_IN_PROGRESS within timeout
* Continue with the job once stack create returns CREATE_COMPLETE
* Fail job on CREATE_FAILED and cleanup stack
* Notify publisher not delete stack when CREATE_IN_PROGRESS
  or DELETE_IN_PROGRESS
* Improve delete-stale-stacks to search for inactive stacks not being
  used by either releng or sandbox siloes and remove them.
* Delete stacks job will now run every hour to cleanup orphaned systems

Change-Id: Ifdc927f601c07e519cdc502a2fb56fca138c659e
Also-by: Thanh Ha <thanh.ha@linuxfoundation.org>
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/jjb/opendaylight-infra-cleanup-stale-stacks.sh b/jjb/opendaylight-infra-cleanup-stale-stacks.sh
index f44ac51079794a844d02635f17ade872e8ec106f..091e212bc87ae3624edea2b6bbc33f4597fd32d9 100644 (file)
--- a/jjb/opendaylight-infra-cleanup-stale-stacks.sh
+++ b/jjb/opendaylight-infra-cleanup-stale-stacks.sh
@@ -5,12 +5,37 @@ pip install --upgrade pip
 pip install --upgrade python-openstackclient python-heatclient
 pip freeze
 
-DELETE_LIST=(`openstack --os-cloud rackspace stack list -f json | \
-              jq -r '.[] | \
-                     select((."Stack Status" == "CREATE_FAILED") or \
-                            (."Stack Status" == "DELETE_FAILED")) | \
-                     ."Stack Name"'`)
-for i in "${DELETE_LIST[@]}"; do
-    echo "Deleting stack $i"
-    openstack --os-cloud rackspace stack delete --yes $i
+#########################
+## FETCH ACTIVE BUILDS ##
+#########################
+# Make sure we fetch active builds on both the releng and sandbox silos
+ACTIVE_BUILDS=()
+for silo in releng sandbox; do
+    JENKINS_URL="https://jenkins.opendaylight.org/$silo//computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
+    wget --no-verbose -O $silo_builds.json $JENKINS_URL
+    sleep 1  # Need to sleep for 1 second otherwise next line causes script to stall
+    ACTIVE_BUILDS=(${ACTIVE_BUILDS[@]} ` \
+        jq -r '.computer[].executors[].currentExecutable.url' $silo_builds.json \
+        | grep -v null | awk -F'/' '{print $6 "-" $7}'`)
+done
+
+##########################
+## DELETE UNUSED STACKS ##
+##########################
+# Search for stacks taht are not in use by either releng or sandbox silos and
+# delete them.
+OS_STACKS=(`openstack --os-cloud rackspace stack list \
+            -f json -c "Stack Name" -c "Stack Status" \
+            --property "stack_status=CREATE_COMPLETE" \
+            --property "stack_status=DELETE_FAILED" \
+            --property "stack_status=CREATE_FAILED" \
+            | jq -r '.[] | ."Stack Name"'`)
+for stack in ${OS_STACKS[@]}; do
+    if [[ "${ACTIVE_BUILDS[@]}" =~ $stack ]]; then
+        # No need to delete stacks if there exists an active build for them
+        continue
+    else
+        echo "Deleting orphaned stack: $stack"
+        openstack --os-cloud rackspace stack delete --yes $stack
+    fi
 done

diff --git a/jjb/opendaylight-infra-stack.sh b/jjb/opendaylight-infra-stack.sh
index 52d450a7162786651ec17d1b84fae711f495c21e..2dc8eb158451bcb1cc98501f3a9cc59b46b52cc0 100644 (file)
--- a/jjb/opendaylight-infra-stack.sh
+++ b/jjb/opendaylight-infra-stack.sh
@@ -9,9 +9,25 @@ cd /builder/openstack-hot
 
 JOB_SUM=`echo $JOB_NAME | sum | awk '{{ print $1 }}'`
 VM_NAME="$JOB_SUM-$BUILD_NUMBER"
-openstack --os-cloud rackspace stack create --wait --timeout 15 -t {stack-template} -e $WORKSPACE/opendaylight-infra-environment.yaml --parameter "job_name=$VM_NAME" --parameter "silo=$SILO" $STACK_NAME
-OS_STATUS=`openstack --os-cloud rackspace stack show -f json -c stack_status $STACK_NAME | jq -r '.stack_status'`
-if [ "$OS_STATUS" != "CREATE_COMPLETE" ]; then
-    echo "Failed to initialize infrastructure. Quitting..."
-    exit 1
-fi
+openstack --os-cloud rackspace stack create -t {stack-template} -e $WORKSPACE/opendaylight-infra-environment.yaml --parameter "job_name=$VM_NAME" --parameter "silo=$SILO" $STACK_NAME
+
+# seq X refers to waiting for X minutes for OpenStack to return
+# a status that is not CREATE_IN_PROGRESS before giving up.
+OS_TIMEOUT=15  # Minutes to wait for OpenStack VM to come online
+echo "Waiting for $OS_TIMEOUT minutes to create $STACK_NAME."
+for i in `seq $OS_TIMEOUT`; do
+    sleep 60
+    OS_STATUS=`openstack --os-cloud rackspace stack show -f json -c stack_status $STACK_NAME | jq -r '.stack_status'`
+    if [ "$OS_STATUS" == "CREATE_COMPLETE" ]; then
+        echo "Stack initialized on infrastructure successful."
+        break
+    elif [ "$OS_STATUS" == "CREATE_FAILED" ]; then
+        echo "ERROR: Failed to initialize infrastructure. Quitting..."
+        exit 1
+    elif [ "$OS_STATUS" == "CREATE_IN_PROGRESS" ]; then
+        echo "Waiting to initialize infrastructure."
+        continue
+    else
+        echo "Unexpected status: $OS_STATUS"
+    fi
+done

diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index 66932081ccc88e99a8472f61bccc6a36e81165c7..970f054af33d330ed1eddbe1b1370d614d2c8379 100644 (file)
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -309,7 +309,8 @@
             build-timeout: '{build-timeout}'
 
     triggers:
-        - timed: 'H H/6 * * *'
+        # Attempt to clear up stacks every hour in case we have orphaned stacks
+        - timed: '0 * * * *'
 
     builders:
         - shell: !include-raw-escape: opendaylight-infra-cleanup-stale-stacks.sh

diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml
index ef6b63e45d471be20e23263b44f729f9da630238..38f4e5651cab78153f649a7454f66520178fd945 100644 (file)
--- a/jjb/releng-macros.yaml
+++ b/jjb/releng-macros.yaml
@@ -281,9 +281,13 @@
         - postbuildscript:
             builders:
                 - shell: |
+                    #!/bin/bash
                     if [ -d "$WORKSPACE/.venv-openstack" ]; then
                         source $WORKSPACE/.venv-openstack/bin/activate
-                        openstack --os-cloud rackspace stack delete --yes $STACK_NAME
+                        OS_STATUS=`openstack --os-cloud rackspace stack show -f json -c stack_status $STACK_NAME | jq -r '.stack_status'`
+                        if [ "$OS_STATUS" != "CREATE_IN_PROGRESS" ] && [ "$OS_STATUS" != "DELETE_IN_PROGRESS" ]; then
+                            openstack --os-cloud rackspace stack delete --yes $STACK_NAME
+                        fi
                     fi
                 - shell: !include-raw: include-raw-deploy-archives.sh
                 - maven-target: