jjb/odl-openstack-cleanup-stale-stacks.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2017 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 # Cleanup stale stacks in the cloud
  12 # Requires the variable JENKINS_URLS declared in the job as a space separated
  13 # list of Jenkins instances to check for active builds.
  14 echo "---> Cleanup stale stacks"
  15
  16 # shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091
  17 source "/tmp/v/openstack/bin/activate"
  18
  19 stack_in_jenkins() {
  20     # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
  21     # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
  22
  23     STACK_NAME="${1}"
  24
  25     builds=()
  26     for jenkins in "${@:2}"; do
  27         JENKINS_URL="$jenkins/computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
  28         resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
  29         json_data=$(echo "$resp" | head -n1)
  30         #status=$(echo "$resp" | awk 'END {print $NF}')
  31
  32         if [[ "${jenkins}" == *"jenkins."*".org" ]]; then
  33             silo="production"
  34         else
  35             silo=$(echo "$jenkins" | sed 's/\/*$//' | awk -F'/' '{print $NF}')
  36         fi
  37         export silo
  38         # We purposely want to wordsplit here to combine the arrays
  39         # shellcheck disable=SC2206,SC2207
  40         builds=(${builds[@]} $(echo "$json_data" | \
  41             jq -r '.computer[].executors[].currentExecutable.url' \
  42             | grep -v null | awk -F'/' '{print ENVIRON["silo"] "-" $6 "-" $7}')
  43         )
  44     done
  45
  46     if [[ "${builds[*]}" =~ $STACK_NAME ]]; then
  47         return 0
  48     fi
  49
  50     return 1
  51 }
  52
  53 #########################
  54 ## FETCH ACTIVE BUILDS ##
  55 #########################
  56 # Fetch stack list before fetching active builds to minimize race condition
  57 # where we might be try to delete stacks while jobs are trying to start
  58
  59 # We purposely need word splitting here to create the OS_STACKS array.
  60 # shellcheck disable=SC2207
  61 OS_STACKS=($(openstack stack list \
  62             -f value -c "Stack Name" -c "Stack Status" \
  63             --property "stack_status=CREATE_COMPLETE" \
  64             --property "stack_status=DELETE_FAILED" \
  65             --property "stack_status=CREATE_FAILED" \
  66             | awk '{print $1}'))
  67
  68 echo "---> Active stacks"
  69 for stack in "${OS_STACKS[@]}"; do
  70     echo "$stack"
  71 done
  72
  73 ##########################
  74 ## DELETE UNUSED STACKS ##
  75 ##########################
  76 echo "---> Delete orphaned stacks"
  77
  78 # Search for stacks that are not in use by either releng or sandbox silos and
  79 # delete them.
  80 for STACK_NAME in "${OS_STACKS[@]}"; do
  81     echo "Checking if orphaned $STACK_NAME"
  82
  83     # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
  84     # needs to be globbed.
  85     # shellcheck disable=SC2153,SC2086
  86     if stack_in_jenkins "$STACK_NAME" $JENKINS_URLS; then
  87         # No need to delete stacks if there exists an active build for them
  88         continue
  89     else
  90         status=$(openstack stack show -f value -c "stack_status" "$STACK_NAME")
  91         case "$status" in
  92             DELETE_IN_PROGRESS)
  93                 echo "skipping delete, $STACK_NAME is already DELETE in progress."
  94                 continue
  95             ;;
  96             DELETE_FAILED)
  97                 # Abandon is not supported in Vexxhost so let's keep trying to
  98                 # delete for now...
  99                 # echo "Stack delete failed, trying to stack abandon now."
 100                 # openstack stack abandon "$STACK_NAME"
 101                 echo "Deleting orphaned stack: $STACK_NAME"
 102                 openstack stack delete --yes "$STACK_NAME"
 103
 104                 echo "------------------------------------"
 105                 echo "Stack details"
 106                 echo "------------------------------------"
 107                 openstack stack show "$STACK_NAME" -f yaml
 108                 echo "------------------------------------"
 109                 continue
 110             ;;
 111             CREATE_COMPLETE|CREATE_FAILED)
 112                 echo "Deleting orphaned stack: $STACK_NAME"
 113                 openstack stack delete --yes "$STACK_NAME"
 114                 continue
 115             ;;
 116             *)
 117                 continue
 118             ;;
 119         esac
 120     fi
 121 done