From: Thanh Ha <thanh.ha@linuxfoundation.org>
Date: Tue, 20 Nov 2018 04:17:51 +0000 (+0800)
Subject: Switch to using global-jjb openstack-cron job
X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=releng%2Fbuilder.git;a=commitdiff_plain;h=bd88d7ec9935c2489480d44766f000e3563be19b

Switch to using global-jjb openstack-cron job

This is provided by global-jjb now.

Change-Id: Ia0c4a03c19cad54d672846ed1da21e705fceb600
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
---

diff --git a/jjb/odl-openstack-check-image-protection.sh b/jjb/odl-openstack-check-image-protection.sh
deleted file mode 100644
index 76a68e5bc..000000000
--- a/jjb/odl-openstack-check-image-protection.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Checks the image "protected" value and set "True" marker
-#
-# The script is involked by 'builder-verify-image-protection', searches
-# the jjb source code for the images presently used and verifies protection
-# setting. If the image protect setting is not "True", sets the
-# image protect setting to "True" to prevent the image from getting purged
-# by the cleanup old images job.
-echo "---> Check image protection"
-
-declare -a yaml_images
-readarray -t yaml_images <<< "$(grep -r _system_image: --include \*.yaml \
-    | awk -F": " -e '{print $3}' | sed "s:'::;s:'$::;/^$/d" | sort -u)"
-declare -a cfg_images
-readarray -t cfg_images <<< "$(grep -r IMAGE_NAME --include \*.cfg \
-    | awk -F'=' -e '{print $2}' | sort -u)"
-combined=("${yaml_images[@]}" "${cfg_images[@]}")
-declare -a images
-readarray -t images <<< "$(printf '%s\n' "${combined[@]}" | sort -u)"
-
-for image in "${images[@]}"; do
-    os_image_protected=$(openstack --os-cloud "$OS_CLOUD" image show "$image" -f value -c protected)
-    echo "Protected setting for $image: $os_image_protected"
-    if [[ $os_image_protected != True ]]; then
-        echo "Image: $image NOT set as protected, changing the protected value."
-        openstack --os-cloud "$OS_CLOUD" image set --protected "$image"
-    fi
-done
diff --git a/jjb/odl-openstack-cleanup-old-images.sh b/jjb/odl-openstack-cleanup-old-images.sh
deleted file mode 100644
index 9cc82cd82..000000000
--- a/jjb/odl-openstack-cleanup-old-images.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 - 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Removes openstack images older than 30 days in the cloud
-echo "---> Cleanup old images"
-
-lftools openstack --os-cloud vex image cleanup --days=30
diff --git a/jjb/odl-openstack-cleanup-orphaned-nodes.sh b/jjb/odl-openstack-cleanup-orphaned-nodes.sh
deleted file mode 100644
index df598b473..000000000
--- a/jjb/odl-openstack-cleanup-orphaned-nodes.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 - 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-echo "---> Cleanup orphaned servers"
-
-minion_in_jenkins() {
-    # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
-    # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
-
-    MINION="${1}"
-
-    minions=()
-    for jenkins in "${@:2}"; do
-        JENKINS_URL="$jenkins/computer/api/json?tree=computer[displayName]"
-        resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
-        json_data=$(echo "$resp" | head -n1)
-        #status=$(echo "$resp" | awk 'END {print $NF}')
-
-        # We purposely want to wordsplit here to combine the arrays
-        # shellcheck disable=SC2206,SC2207
-        minions=(${minions[@]} $(echo "$json_data" | \
-            jq -r '.computer[].displayName' | grep -v master)
-        )
-    done
-
-    if [[ "${minions[*]}" =~ $MINION ]]; then
-        return 0
-    fi
-
-    return 1
-}
-
-##########################
-## FETCH ACTIVE MINIONS ##
-##########################
-# Fetch server list before fetching active minions to minimize race condition
-# where we might be trying to delete servers while jobs are trying to start
-
-# We purposely need word splitting here to create the OS_SERVERS array.
-# shellcheck disable=SC2207
-mapfile -t OS_SERVERS < <(openstack server list -f value -c "Name" | grep -E 'prd|snd')
-
-#############################
-## DELETE ORPHANED SERVERS ##
-#############################
-
-# Search for servers that are not in use by either releng or sandbox silos and
-# delete them.
-for server in "${OS_SERVERS[@]}"; do
-    # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
-    # needs to be globbed.
-    # shellcheck disable=SC2153,SC2086
-    if minion_in_jenkins "$server" $JENKINS_URLS; then
-        # No need to delete server if it is still attached to Jenkins
-        continue
-    else
-        echo "Deleting $server"
-        lftools openstack --os-cloud vex \
-            server remove --minutes 15 "$server"
-    fi
-done
diff --git a/jjb/odl-openstack-cleanup-stale-nodes.sh b/jjb/odl-openstack-cleanup-stale-nodes.sh
deleted file mode 100644
index 25cf88c70..000000000
--- a/jjb/odl-openstack-cleanup-stale-nodes.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 - 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-echo "---> Cleanup stale nodes"
-
-# Todo: As a safe check we could obtain the list of active jobs from Jenkins and
-# compute the checksum from $JOB_NAME to check if any active nodes exist and
-# skip deleting those nodes. This step may not be required since there is already
-# 24H timeout in place for all jobs therefore all jobs are expected to complete
-# within the timeout.
-
-lftools openstack --os-cloud vex server list --days=1
-lftools openstack --os-cloud vex server cleanup --days=1
diff --git a/jjb/odl-openstack-cleanup-stale-stacks.sh b/jjb/odl-openstack-cleanup-stale-stacks.sh
deleted file mode 100644
index 6d5e553c1..000000000
--- a/jjb/odl-openstack-cleanup-stale-stacks.sh
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Cleanup stale stacks in the cloud
-# Requires the variable JENKINS_URLS declared in the job as a space separated
-# list of Jenkins instances to check for active builds.
-echo "---> Cleanup stale stacks"
-
-stack_in_jenkins() {
-    # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
-    # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
-
-    STACK_NAME="${1}"
-
-    builds=()
-    for jenkins in "${@:2}"; do
-        JENKINS_URL="$jenkins/computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
-        resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
-        json_data=$(echo "$resp" | head -n1)
-        #status=$(echo "$resp" | awk 'END {print $NF}')
-
-        if [[ "${jenkins}" == *"jenkins."*".org" ]]; then
-            silo="production"
-        else
-            silo=$(echo "$jenkins" | sed 's/\/*$//' | awk -F'/' '{print $NF}')
-        fi
-        export silo
-        # We purposely want to wordsplit here to combine the arrays
-        # shellcheck disable=SC2206,SC2207
-        builds=(${builds[@]} $(echo "$json_data" | \
-            jq -r '.computer[].executors[].currentExecutable.url' \
-            | grep -v null | awk -F'/' '{print ENVIRON["silo"] "-" $6 "-" $7}')
-        )
-    done
-
-    if [[ "${builds[*]}" =~ $STACK_NAME ]]; then
-        return 0
-    fi
-
-    return 1
-}
-
-#########################
-## FETCH ACTIVE BUILDS ##
-#########################
-# Fetch stack list before fetching active builds to minimize race condition
-# where we might be try to delete stacks while jobs are trying to start
-
-# We purposely need word splitting here to create the OS_STACKS array.
-# shellcheck disable=SC2207
-OS_STACKS=($(openstack stack list \
-            -f value -c "Stack Name" -c "Stack Status" \
-            --property "stack_status=CREATE_COMPLETE" \
-            --property "stack_status=DELETE_FAILED" \
-            --property "stack_status=CREATE_FAILED" \
-            | awk '{print $1}'))
-
-echo "---> Active stacks"
-for stack in "${OS_STACKS[@]}"; do
-    echo "$stack"
-done
-
-##########################
-## DELETE UNUSED STACKS ##
-##########################
-echo "---> Delete orphaned stacks"
-
-# Search for stacks that are not in use by either releng or sandbox silos and
-# delete them.
-for STACK_NAME in "${OS_STACKS[@]}"; do
-    echo "Checking if orphaned $STACK_NAME"
-
-    # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
-    # needs to be globbed.
-    # shellcheck disable=SC2153,SC2086
-    if stack_in_jenkins "$STACK_NAME" $JENKINS_URLS; then
-        # No need to delete stacks if there exists an active build for them
-        continue
-    else
-        status=$(openstack stack show -f value -c "stack_status" "$STACK_NAME")
-        case "$status" in
-            DELETE_IN_PROGRESS)
-                echo "skipping delete, $STACK_NAME is already DELETE in progress."
-                continue
-            ;;
-            DELETE_FAILED)
-                # Abandon is not supported in Vexxhost so let's keep trying to
-                # delete for now...
-                # echo "Stack delete failed, trying to stack abandon now."
-                # openstack stack abandon "$STACK_NAME"
-                echo "Deleting orphaned stack: $STACK_NAME"
-                openstack stack delete --yes "$STACK_NAME"
-
-                echo "------------------------------------"
-                echo "Stack details"
-                echo "------------------------------------"
-                openstack stack show "$STACK_NAME" -f yaml
-                echo "------------------------------------"
-                continue
-            ;;
-            CREATE_COMPLETE|CREATE_FAILED)
-                echo "Deleting orphaned stack: $STACK_NAME"
-                openstack stack delete --yes "$STACK_NAME"
-                continue
-            ;;
-            *)
-                continue
-            ;;
-        esac
-    fi
-done
diff --git a/jjb/odl-openstack-cleanup-stale-volumes.sh b/jjb/odl-openstack-cleanup-stale-volumes.sh
deleted file mode 100644
index 3745eb253..000000000
--- a/jjb/odl-openstack-cleanup-stale-volumes.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Scans OpenStack for orphaned volumes
-
-mapfile -t os_volumes < <(openstack volume list -f value -c ID --status Available)
-
-echo "---> Orphaned volumes"
-if [ ${#os_volumes[@]} -eq 0 ]; then
-    echo "No orphaned volumes found."
-else
-    for volume in "${os_volumes[@]}"; do
-        echo "Removing volume $volume"
-        lftools openstack --os-cloud vex volume remove --minutes 15 "$volume"
-    done
-fi
diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index a63ef7eb8..2e92ff82d 100644
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -16,7 +16,6 @@
       - gerrit-jjb-verify
       - builder-check-poms
       - builder-copy-sandbox-logs
-      - builder-openstack-cron
       - gerrit-tox-verify
       # Automation for docs and jobs
       - 'builder-update-image-list'
@@ -35,6 +34,20 @@
       https://jenkins.opendaylight.org/releng
       https://jenkins.opendaylight.org/sandbox
 
+- project:
+    name: builder-openstack
+    jobs:
+      - gerrit-openstack-cron
+
+    project: 'releng/builder'
+    project-name: builder
+    build-node: centos7-builder-2c-2g
+
+    jenkins-urls: >
+        https://jenkins.opendaylight.org
+        https://jenkins.opendaylight.org/sandbox
+    openstack-cloud: vex
+
 - job-template:
     name: builder-check-poms
     node: centos7-builder-2c-1g
@@ -111,79 +124,6 @@
       - lf-infra-publish
 
 
-- job-template:
-    name: builder-openstack-cron
-    project-type: freestyle
-    node: '{build-node}'
-    concurrent: true
-
-    properties:
-      - lf-infra-properties:
-          build-days-to-keep: 7
-
-    parameters:
-      - lf-infra-parameters:
-          project: '{project}'
-          stream: '{stream}'
-          branch: '{branch}'
-          lftools-version: '{lftools-version}'
-      - string:
-          name: OS_CLOUD
-          default: '{os-cloud}'
-          description: |
-              The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
-              variable name that is significant to openstack client as a
-              environment variable. Please refer to the documentation for
-              further details.
-              https://docs.openstack.org/developer/python-openstackclient/
-      - string:
-          name: ARCHIVE_ARTIFACTS
-          default: '{archive-artifacts}'
-          description: Artifacts to archive to the logs server.
-      - string:
-          name: JENKINS_URLS
-          default: '{jenkins-urls}'
-          description: 'Space separated list of Jenkins URLs to check for active builds'
-
-    scm:
-      - git-scm:
-          branch: '{branch}'
-
-    wrappers:
-      - lf-infra-wrappers:
-          build-timeout: '{build-timeout}'
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-      # Listed after to override openstack-infra-wrappers clouds.yaml definition
-      - config-file-provider:
-          files:
-            - file-id: clouds-yaml
-              target: '$HOME/.config/openstack/clouds.yaml'
-            - file-id: npmrc
-              target: '$HOME/.npmrc'
-            - file-id: pipconf
-              target: '$HOME/.config/pip/pip.conf'
-
-    triggers:
-      - timed: '0,30 * * * *'
-
-    builders:
-      - lf-infra-pre-build
-      - shell: |
-          #!/bin/bash -l
-          pip install --user --upgrade lftools[openstack]~=0.17.1
-      # Servers
-      - odl-openstack-cleanup-stale-stacks
-      - odl-openstack-cleanup-stale-nodes
-      - odl-openstack-cleanup-orphaned-nodes
-      # Volumes
-      - odl-openstack-cleanup-stale-volumes
-      # Images
-      - odl-openstack-check-image-protection
-      - odl-openstack-cleanup-old-images
-
-    publishers:
-      - lf-infra-publish
-
 - job-template:
     name: 'builder-update-image-list'
     project-type: freestyle
diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml
index 4a850e4c1..7c0a91643 100644
--- a/jjb/releng-macros.yaml
+++ b/jjb/releng-macros.yaml
@@ -247,36 +247,6 @@
           send-to:
             - recipients
 
-- builder:
-    name: odl-openstack-check-image-protection
-    builders:
-      - shell: !include-raw: odl-openstack-check-image-protection.sh
-
-- builder:
-    name: odl-openstack-cleanup-old-images
-    builders:
-      - shell: !include-raw: odl-openstack-cleanup-old-images.sh
-
-- builder:
-    name: odl-openstack-cleanup-orphaned-nodes
-    builders:
-      - shell: !include-raw: odl-openstack-cleanup-orphaned-nodes.sh
-
-- builder:
-    name: odl-openstack-cleanup-stale-nodes
-    builders:
-      - shell: !include-raw: odl-openstack-cleanup-stale-nodes.sh
-
-- builder:
-    name: odl-openstack-cleanup-stale-stacks
-    builders:
-      - shell: !include-raw: odl-openstack-cleanup-stale-stacks.sh
-
-- builder:
-    name: odl-openstack-cleanup-stale-volumes
-    builders:
-      - shell: !include-raw: odl-openstack-cleanup-stale-volumes.sh
-
 - builder:
     name: opendaylight-infra-stack
     # opendaylight-infra-stack.sh has a required variable {stack-template} that