jjb/opendaylight-infra-cleanup-stale-stacks.sh

   1 #!/bin/bash
   2 virtualenv "$WORKSPACE/.venv"
   3 # shellcheck disable=SC1090
   4 source "$WORKSPACE/.venv/bin/activate"
   5 PYTHON="$WORKSPACE/.venv/bin/python"
   6 $PYTHON -m pip install --upgrade pip
   7 $PYTHON -m pip install --upgrade python-openstackclient python-heatclient
   8 $PYTHON -m pip freeze
   9
  10 #########################
  11 ## FETCH ACTIVE BUILDS ##
  12 #########################
  13 # Fetch stack list before fetching active builds to minimize race condition
  14 # where we might be try to delete stacks while jobs are trying to start
  15 OS_STACKS=($(openstack stack list \
  16             -f json -c "Stack Name" -c "Stack Status" \
  17             --property "stack_status=CREATE_COMPLETE" \
  18             --property "stack_status=DELETE_FAILED" \
  19             --property "stack_status=CREATE_FAILED" \
  20             | jq -r '.[] | ."Stack Name"'))
  21
  22 # Make sure we fetch active builds on both the releng and sandbox silos
  23 ACTIVE_BUILDS=()
  24 for silo in releng sandbox; do
  25     JENKINS_URL="https://jenkins.opendaylight.org/$silo//computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
  26     wget --no-verbose -O "${silo}_builds.json" "$JENKINS_URL"
  27     sleep 1  # Need to sleep for 1 second otherwise next line causes script to stall
  28     ACTIVE_BUILDS=(${ACTIVE_BUILDS[@]} $( \
  29         jq -r '.computer[].executors[].currentExecutable.url' "${silo}_builds.json" \
  30         | grep -v null | awk -F'/' '{print $4 "-" $6 "-" $7}'))
  31 done
  32
  33 ##########################
  34 ## DELETE UNUSED STACKS ##
  35 ##########################
  36 # Search for stacks that are not in use by either releng or sandbox silos and
  37 # delete them.
  38 for STACK_NAME in "${OS_STACKS[@]}"; do
  39     STACK_STATUS=$(openstack stack show -f json -c "stack_status" "$STACK_NAME" | jq -r '."stack_status"')
  40     if [[ "${ACTIVE_BUILDS[@]}" =~ $STACK_NAME ]]; then
  41         # No need to delete stacks if there exists an active build for them
  42         continue
  43     else
  44         case "$STACK_STATUS" in
  45             DELETE_IN_PROGRESS)
  46                 echo "skipping delete, $STACK_NAME is already DELETE in progress."
  47                 continue
  48             ;;
  49             DELETE_FAILED)
  50                 echo "Stack delete failed, trying to stack abandon now."
  51                 # stack abandon does not work on RS, therefore requires acquiring a token
  52                 # and using http delete method to abondon DELETE_FAILED stacks
  53                 # Todo: remove the change once RS fixes the issue upstream
  54                 # openstack stack abandon "$STACK_NAME"
  55                 STACK_ID=$(openstack stack show -f json -c "id" "$STACK_NAME" | jq -r '."id"')
  56                 TOKEN=$(openstack token issue -f json -c id | jq -r '.id')
  57                 curl -si -X DELETE -H "Content-Type: application/json" -H "Accept: application/json"\
  58                     -H "x-auth-token: $TOKEN"\
  59                     "https://dfw.orchestration.api.rackspacecloud.com/v1/904885/stacks/$STACK_NAME/$STACK_ID/abandon"
  60                 STACK_SHOW=$(openstack stack show "$STACK_NAME")
  61                 echo "$STACK_SHOW"
  62                 continue
  63             ;;
  64             CREATE_COMPLETE|CREATE_FAILED)
  65                 echo "Deleting orphaned stack: $STACK_NAME"
  66                 openstack stack delete --yes "$STACK_NAME"
  67                 continue
  68             ;;
  69             *)
  70                 continue
  71             ;;
  72         esac
  73     fi
  74 done