Add a job to cleanup orphaned nodes

This job assumes that releng minions have a 'prd' prefix and
sandbox has a 'snd' prefix. These minions are Jenkins managed
minions and are not Heat managed minions.

The script get's a list of jobs with the prd or snd prefix and then
checks against Jenkins to see if the nodes are attached to Jenkins
if they are not then issue a delete call as it is likely Jenkins
lost track of them.

Issue: RELENG-724
Change-Id: I624102ace279d49d2696d287cfea9e4e89ba7a20
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh b/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh
new file mode 100644 (file)
index 0000000..47c78c6
--- /dev/null
+++ b/jjb/opendaylight-infra-cleanup-orphaned-nodes.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo "---> Cleanup orphaned servers"
+
+virtualenv "/tmp/v/openstack"
+# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091
+source "/tmp/v/openstack/bin/activate"
+pip install --upgrade pip
+pip install --upgrade python-openstackclient python-heatclient
+pip install --upgrade pipdeptree
+pipdeptree
+
+##########################
+## FETCH ACTIVE MINIONS ##
+##########################
+# Fetch server list before fetching active minions to minimize race condition
+# where we might be try to delete servers while jobs are trying to start
+OS_SERVERS=($(openstack server list -f value -c "Name" | grep -E 'prd|snd'))
+
+# Make sure we fetch active minions on both the releng and sandbox silos
+ACTIVE_MINIONS=()
+for silo in releng sandbox; do
+    JENKINS_URL="https://jenkins.opendaylight.org/$silo/computer/api/json?tree=computer[displayName]"
+    wget -nv -O "${silo}_builds.json" "$JENKINS_URL"
+    sleep 1  # Need to sleep for 1 second otherwise next line causes script to stall
+    ACTIVE_MINIONS=(${ACTIVE_MINIONS[@]} $( \
+        jq -r '.computer[].displayName' "${silo}_builds.json" | grep -v master))
+done
+
+#############################
+## DELETE ORPHANED SERVERS ##
+#############################
+# Search for servers that are not in use by either releng or sandbox silos and
+# delete them.
+for server in "${OS_SERVERS[@]}"; do
+    if [[ "${ACTIVE_MINIONS[*]}" =~ $server ]]; then
+        # No need to delete server if it is still attached to Jenkins
+        continue
+    else
+        echo "Deleting $server"
+        openstack server delete "$server"
+    fi
+done

diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index ebfaf09400675c99a63ae9517e58c6340b1f7c23..ce56ee1e4236e7eeba0048801cb25bafc3aaf759 100644 (file)
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -8,6 +8,7 @@
       # OpenStack Related
       - 'builder-verify-image-protection'
       - 'builder-cleanup-old-images'
       # OpenStack Related
       - 'builder-verify-image-protection'
       - 'builder-cleanup-old-images'

+      - 'builder-delete-orphaned-nodes'

       - 'builder-delete-stale-nodes'
       - 'builder-delete-stale-stacks'
       # Automation for docs and jobs
       - 'builder-delete-stale-nodes'
       - 'builder-delete-stale-stacks'
       # Automation for docs and jobs


@@ -232,6 +233,41 @@
       - lf-infra-publish
 
 
       - lf-infra-publish
 
 

+- job-template:
+    name: builder-delete-orphaned-nodes
+    project-type: freestyle
+    node: centos7-java-builder-1c-4g
+
+    properties:
+      - opendaylight-infra-properties:
+          build-days-to-keep: 7
+
+    parameters:
+      - opendaylight-infra-parameters:
+          os-cloud: '{os-cloud}'
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+          artifacts: '{archive-artifacts}'
+
+    wrappers:
+      - opendaylight-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      # Attempt to clear up servers every 30 mins in case we have orphaned servers
+      - timed: '0,30 * * * *'
+
+    builders:
+      - shell: !include-raw-escape: opendaylight-infra-cleanup-orphaned-nodes.sh
+
+    publishers:
+      - email-notification:
+          email-recipients: '{email-recipients}'
+          email-prefix: '[releng]'
+      - lf-infra-publish
+
+

 - job-template:
     name: 'builder-delete-stale-nodes'
     project-type: freestyle
 - job-template:
     name: 'builder-delete-stale-nodes'
     project-type: freestyle