# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
##############################################################################
+# Cleanup stale stacks in the cloud
+# Requires the variable JENKINS_URLS declared in the job as a space separated
+# list of Jenkins instances to check for active builds.
virtualenv "/tmp/v/openstack"
# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091
pip install --upgrade pipdeptree
pipdeptree
+stack_in_jenkins() {
+ # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
+ # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
+
+ STACK_NAME="${1}"
+
+ builds=()
+ for jenkins in "${@:2}"; do
+ JENKINS_URL="$jenkins/computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
+ resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
+ json_data=$(echo "$resp" | head -n1)
+ #status=$(echo "$resp" | awk 'END {print $NF}')
+
+ if [[ "${jenkins}" == *"jenkins."*".org" ]]; then
+ silo="production"
+ else
+ silo=$(echo "$jenkins" | sed 's/\/*$//' | awk -F'/' '{print $NF}')
+ fi
+ export silo
+ # We purposely want to wordsplit here to combine the arrays
+ # shellcheck disable=SC2206,SC2207
+ builds=(${builds[@]} $(echo "$json_data" | \
+ jq -r '.computer[].executors[].currentExecutable.url' \
+ | grep -v null | awk -F'/' '{print ENVIRON["silo"] "-" $6 "-" $7}')
+ )
+ done
+
+ if [[ "${builds[*]}" =~ $STACK_NAME ]]; then
+ return 0
+ fi
+
+ return 1
+}
+
#########################
## FETCH ACTIVE BUILDS ##
#########################
# Fetch stack list before fetching active builds to minimize race condition
# where we might be try to delete stacks while jobs are trying to start
+
+# We purposely need word splitting here to create the OS_STACKS array.
+# shellcheck disable=SC2207
OS_STACKS=($(openstack stack list \
- -f json -c "Stack Name" -c "Stack Status" \
+ -f value -c "Stack Name" -c "Stack Status" \
--property "stack_status=CREATE_COMPLETE" \
--property "stack_status=DELETE_FAILED" \
--property "stack_status=CREATE_FAILED" \
- | jq -r '.[] | ."Stack Name"'))
+ | awk '{print $1}'))
-# Make sure we fetch active builds on both the releng and sandbox silos
-ACTIVE_BUILDS=()
-for silo in releng sandbox; do
- JENKINS_URL="https://jenkins.opendaylight.org/$silo//computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
- wget -nv -O "${silo}_builds.json" "$JENKINS_URL"
- sleep 1 # Need to sleep for 1 second otherwise next line causes script to stall
- ACTIVE_BUILDS=(${ACTIVE_BUILDS[@]} $( \
- jq -r '.computer[].executors[].currentExecutable.url' "${silo}_builds.json" \
- | grep -v null | awk -F'/' '{print $4 "-" $6 "-" $7}'))
+echo "---> Active stacks"
+for stack in "${OS_STACKS[@]}"; do
+ echo "$stack"
done
##########################
## DELETE UNUSED STACKS ##
##########################
+echo "---> Delete orphaned stacks"
+
# Search for stacks that are not in use by either releng or sandbox silos and
# delete them.
for STACK_NAME in "${OS_STACKS[@]}"; do
- STACK_STATUS=$(openstack stack show -f json -c "stack_status" "$STACK_NAME" | jq -r '."stack_status"')
- if [[ "${ACTIVE_BUILDS[*]}" =~ $STACK_NAME ]]; then
+ echo "Checking if orphaned $STACK_NAME"
+
+ # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
+ # needs to be globbed.
+ # shellcheck disable=SC2153,SC2086
+ if stack_in_jenkins "$STACK_NAME" $JENKINS_URLS; then
# No need to delete stacks if there exists an active build for them
continue
else
- case "$STACK_STATUS" in
+ status=$(openstack stack show -f value -c "stack_status" "$STACK_NAME")
+ case "$status" in
DELETE_IN_PROGRESS)
echo "skipping delete, $STACK_NAME is already DELETE in progress."
continue
;;
DELETE_FAILED)
- echo "Stack delete failed, trying to stack abandon now."
- # stack abandon does not work on RS, therefore requires acquiring a token
- # and using http delete method to abondon DELETE_FAILED stacks
- # Todo: remove the change once RS fixes the issue upstream
+ # Abandon is not supported in Vexxhost so let's keep trying to
+ # delete for now...
+ # echo "Stack delete failed, trying to stack abandon now."
# openstack stack abandon "$STACK_NAME"
- STACK_ID=$(openstack stack show -f json -c "id" "$STACK_NAME" | jq -r '."id"')
- TOKEN=$(openstack token issue -f json -c id | jq -r '.id')
- curl -si -X DELETE -H "Content-Type: application/json" -H "Accept: application/json"\
- -H "x-auth-token: $TOKEN"\
- "https://dfw.orchestration.api.rackspacecloud.com/v1/904885/stacks/$STACK_NAME/$STACK_ID/abandon"
- STACK_SHOW=$(openstack stack show "$STACK_NAME")
- echo "$STACK_SHOW"
+ echo "Deleting orphaned stack: $STACK_NAME"
+ openstack stack delete --yes "$STACK_NAME"
+
+ echo "------------------------------------"
+ echo "Stack details"
+ echo "------------------------------------"
+ openstack stack show "$STACK_NAME" -f yaml
+ echo "------------------------------------"
continue
;;
CREATE_COMPLETE|CREATE_FAILED)
Code Review / releng / builder.git / blobdiff