Check for stacks delete in progress

[releng/builder.git] / jjb / opendaylight-infra-cleanup-stale-stacks.sh

diff --git a/jjb/opendaylight-infra-cleanup-stale-stacks.sh b/jjb/opendaylight-infra-cleanup-stale-stacks.sh
index db2390bdf1c42ecdb717479f81f90b334722900e..d99938b3444dbbc09a741e9dc4305eba0c4334f9 100644 (file)
--- a/jjb/opendaylight-infra-cleanup-stale-stacks.sh
+++ b/jjb/opendaylight-infra-cleanup-stale-stacks.sh
@@ -26,20 +26,49 @@ for silo in releng sandbox; do
     sleep 1  # Need to sleep for 1 second otherwise next line causes script to stall
     ACTIVE_BUILDS=(${ACTIVE_BUILDS[@]} $( \
         jq -r '.computer[].executors[].currentExecutable.url' "${silo}_builds.json" \
-        | grep -v null | awk -F'/' '{print $6 "-" $7}'))
+        | grep -v null | awk -F'/' '{print $4 "-" $6 "-" $7}'))
 done
 
 ##########################
 ## DELETE UNUSED STACKS ##
 ##########################
-# Search for stacks taht are not in use by either releng or sandbox silos and
+# Search for stacks that are not in use by either releng or sandbox silos and
 # delete them.
-for stack in "${OS_STACKS[@]}"; do
-    if [[ "${ACTIVE_BUILDS[@]}" =~ $stack ]]; then
+for STACK_NAME in "${OS_STACKS[@]}"; do
+    STACK_STATUS=$(openstack stack show -f json -c "stack_status" "$STACK_NAME" | jq -r '."stack_status"')
+    if [[ "${ACTIVE_BUILDS[@]}" =~ $STACK_NAME ]]; then
         # No need to delete stacks if there exists an active build for them
         continue
     else
-        echo "Deleting orphaned stack: $stack"
-        openstack stack delete --yes "$stack"
+        OS_STATUS=$(openstack stack show -f json -c stack_status "$stack" | jq -r '.stack_status')
+        case "$OS_STATUS" in
+            DELETE_IN_PROGRESS)
+                echo "skipping delete, $stack is already DELETE in progress."
+                continue
+            ;;
+            DELETE_FAILED)
+                echo "Stack delete failed, trying to stack abandon now."
+                # stack abandon does not work on RS, therefore requires acquiring a token
+                # and using http delete method to abondon DELETE_FAILED stacks
+                # Todo: remove the change once RS fixes the issue upstream
+                # openstack stack abandon "$STACK_NAME"
+                STACK_ID=$(openstack stack show -f json -c "id" "$STACK_NAME" | jq -r '."id"')
+                TOKEN=$(openstack token issue -f json -c id | jq -r '.id')
+                curl -si -X DELETE -H "Content-Type: application/json" -H "Accept: application/json"\
+                    -H "x-auth-token: $TOKEN"\
+                    "https://dfw.orchestration.api.rackspacecloud.com/v1/904885/stacks/$STACK_NAME/$STACK_ID/abandon"
+                STACK_SHOW=$(openstack stack show "$STACK_NAME")
+                echo "$STACK_SHOW"
+                continue
+            ;;
+            CREATE_COMPLETE|CREATE_FAILED)
+                echo "Deleting orphaned stack: $stack"
+                openstack stack delete --yes "$stack"
+                continue
+            ;;
+            *)
+                continue
+            ;;
+        esac
     fi
 done