Improve orphaned-nodes to check on each delete

Similar to I2fcc0a9bef34708062e69f4d8a4ee04dfd6108b5 improve the
script here to check closer to the delete to reduce the race
condition timing.

Issue: RELENG-75
Change-Id: I8aff148efc11ede656747978ba570075ce679e3a
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh b/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh
index b999e30cb5dccd8afc85518fac12b5c3ac09314d..1a832fe092782c4f3dd0d7f32c8cc0ba57e96d29 100644 (file)
--- a/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh
+++ b/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh
@@ -18,22 +18,42 @@ pip install --upgrade python-openstackclient python-heatclient
 pip install --upgrade pipdeptree
 pipdeptree
 
+minion_in_jenkins() {
+    # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
+    # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
+
+    MINION="${1}"
+
+    minions=()
+    for jenkins in "${@:2}"; do
+        JENKINS_URL="$jenkins/computer/api/json?tree=computer[displayName]"
+        resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
+        json_data=$(echo "$resp" | head -n1)
+        #status=$(echo "$resp" | awk 'END {print $NF}')
+
+        # We purposely want to wordsplit here to combine the arrays
+        # shellcheck disable=SC2206,SC2207
+        minions=(${minions[@]} $(echo "$json_data" | \
+            jq -r '.computer[].displayName' | grep -v master)
+        )
+    done
+
+    if [[ "${minions[*]}" =~ $MINION ]]; then
+        return 0
+    fi
+
+    return 1
+}
+
 ##########################
 ## FETCH ACTIVE MINIONS ##
 ##########################
 # Fetch server list before fetching active minions to minimize race condition
 # where we might be trying to delete servers while jobs are trying to start
-OS_SERVERS=($(openstack server list -f value -c "Name" | grep -E 'prd|snd'))
 
-# Make sure we fetch active minions on both the releng and sandbox silos
-ACTIVE_MINIONS=()
-for silo in releng sandbox; do
-    JENKINS_URL="https://jenkins.opendaylight.org/$silo/computer/api/json?tree=computer[displayName]"
-    wget -nv -O "${silo}_builds.json" "$JENKINS_URL"
-    sleep 1  # Need to sleep for 1 second otherwise next line causes script to stall
-    ACTIVE_MINIONS=(${ACTIVE_MINIONS[@]} $( \
-        jq -r '.computer[].displayName' "${silo}_builds.json" | grep -v master))
-done
+# We purposely need word splitting here to create the OS_SERVERS array.
+# shellcheck disable=SC2207
+OS_SERVERS=($(openstack server list -f value -c "Name" | grep -E 'prd|snd'))
 
 #############################
 ## DELETE ORPHANED SERVERS ##
@@ -41,7 +61,10 @@ done
 # Search for servers that are not in use by either releng or sandbox silos and
 # delete them.
 for server in "${OS_SERVERS[@]}"; do
-    if [[ "${ACTIVE_MINIONS[*]}" =~ $server ]]; then
+    # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
+    # needs to be globbed.
+    # shellcheck disable=SC2153,SC2086
+    if minion_in_jenkins "$server" $JENKINS_URLS; then
         # No need to delete server if it is still attached to Jenkins
         continue
     else

diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index 59aef4dfcb81988ed44ce76f7ffc6b6db4ddb3d4..e606688a3942bd40182fcb558f84a01dab86dcad 100644 (file)
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -249,6 +249,10 @@
           branch: '{branch}'
           refspec: 'refs/heads/{branch}'
           artifacts: '{archive-artifacts}'
+      - string:
+          name: JENKINS_URLS
+          default: 'https://jenkins.opendaylight.org/releng https://jenkins.opendaylight.org/sandbox'
+          description: 'Space separated list of Jenkins URLs to check for active builds'
 
     wrappers:
       - opendaylight-infra-wrappers: