This is provided by global-jjb now.
Change-Id: Ia0c4a03c19cad54d672846ed1da21e705fceb600
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
+++ /dev/null
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Checks the image "protected" value and set "True" marker
-#
-# The script is involked by 'builder-verify-image-protection', searches
-# the jjb source code for the images presently used and verifies protection
-# setting. If the image protect setting is not "True", sets the
-# image protect setting to "True" to prevent the image from getting purged
-# by the cleanup old images job.
-echo "---> Check image protection"
-
-declare -a yaml_images
-readarray -t yaml_images <<< "$(grep -r _system_image: --include \*.yaml \
- | awk -F": " -e '{print $3}' | sed "s:'::;s:'$::;/^$/d" | sort -u)"
-declare -a cfg_images
-readarray -t cfg_images <<< "$(grep -r IMAGE_NAME --include \*.cfg \
- | awk -F'=' -e '{print $2}' | sort -u)"
-combined=("${yaml_images[@]}" "${cfg_images[@]}")
-declare -a images
-readarray -t images <<< "$(printf '%s\n' "${combined[@]}" | sort -u)"
-
-for image in "${images[@]}"; do
- os_image_protected=$(openstack --os-cloud "$OS_CLOUD" image show "$image" -f value -c protected)
- echo "Protected setting for $image: $os_image_protected"
- if [[ $os_image_protected != True ]]; then
- echo "Image: $image NOT set as protected, changing the protected value."
- openstack --os-cloud "$OS_CLOUD" image set --protected "$image"
- fi
-done
+++ /dev/null
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 - 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Removes openstack images older than 30 days in the cloud
-echo "---> Cleanup old images"
-
-lftools openstack --os-cloud vex image cleanup --days=30
+++ /dev/null
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 - 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-echo "---> Cleanup orphaned servers"
-
-minion_in_jenkins() {
- # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
- # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
-
- MINION="${1}"
-
- minions=()
- for jenkins in "${@:2}"; do
- JENKINS_URL="$jenkins/computer/api/json?tree=computer[displayName]"
- resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
- json_data=$(echo "$resp" | head -n1)
- #status=$(echo "$resp" | awk 'END {print $NF}')
-
- # We purposely want to wordsplit here to combine the arrays
- # shellcheck disable=SC2206,SC2207
- minions=(${minions[@]} $(echo "$json_data" | \
- jq -r '.computer[].displayName' | grep -v master)
- )
- done
-
- if [[ "${minions[*]}" =~ $MINION ]]; then
- return 0
- fi
-
- return 1
-}
-
-##########################
-## FETCH ACTIVE MINIONS ##
-##########################
-# Fetch server list before fetching active minions to minimize race condition
-# where we might be trying to delete servers while jobs are trying to start
-
-# We purposely need word splitting here to create the OS_SERVERS array.
-# shellcheck disable=SC2207
-mapfile -t OS_SERVERS < <(openstack server list -f value -c "Name" | grep -E 'prd|snd')
-
-#############################
-## DELETE ORPHANED SERVERS ##
-#############################
-
-# Search for servers that are not in use by either releng or sandbox silos and
-# delete them.
-for server in "${OS_SERVERS[@]}"; do
- # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
- # needs to be globbed.
- # shellcheck disable=SC2153,SC2086
- if minion_in_jenkins "$server" $JENKINS_URLS; then
- # No need to delete server if it is still attached to Jenkins
- continue
- else
- echo "Deleting $server"
- lftools openstack --os-cloud vex \
- server remove --minutes 15 "$server"
- fi
-done
+++ /dev/null
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 - 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-echo "---> Cleanup stale nodes"
-
-# Todo: As a safe check we could obtain the list of active jobs from Jenkins and
-# compute the checksum from $JOB_NAME to check if any active nodes exist and
-# skip deleting those nodes. This step may not be required since there is already
-# 24H timeout in place for all jobs therefore all jobs are expected to complete
-# within the timeout.
-
-lftools openstack --os-cloud vex server list --days=1
-lftools openstack --os-cloud vex server cleanup --days=1
+++ /dev/null
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Cleanup stale stacks in the cloud
-# Requires the variable JENKINS_URLS declared in the job as a space separated
-# list of Jenkins instances to check for active builds.
-echo "---> Cleanup stale stacks"
-
-stack_in_jenkins() {
- # Usage: check_stack_in_jenkins STACK_NAME JENKINS_URL [JENKINS_URL...]
- # Returns: 0 If stack is in Jenkins and 1 if stack is not in Jenkins.
-
- STACK_NAME="${1}"
-
- builds=()
- for jenkins in "${@:2}"; do
- JENKINS_URL="$jenkins/computer/api/json?tree=computer[executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]]&xpath=//url&wrapper=builds"
- resp=$(curl -s -w "\\n\\n%{http_code}" --globoff -H "Content-Type:application/json" "$JENKINS_URL")
- json_data=$(echo "$resp" | head -n1)
- #status=$(echo "$resp" | awk 'END {print $NF}')
-
- if [[ "${jenkins}" == *"jenkins."*".org" ]]; then
- silo="production"
- else
- silo=$(echo "$jenkins" | sed 's/\/*$//' | awk -F'/' '{print $NF}')
- fi
- export silo
- # We purposely want to wordsplit here to combine the arrays
- # shellcheck disable=SC2206,SC2207
- builds=(${builds[@]} $(echo "$json_data" | \
- jq -r '.computer[].executors[].currentExecutable.url' \
- | grep -v null | awk -F'/' '{print ENVIRON["silo"] "-" $6 "-" $7}')
- )
- done
-
- if [[ "${builds[*]}" =~ $STACK_NAME ]]; then
- return 0
- fi
-
- return 1
-}
-
-#########################
-## FETCH ACTIVE BUILDS ##
-#########################
-# Fetch stack list before fetching active builds to minimize race condition
-# where we might be try to delete stacks while jobs are trying to start
-
-# We purposely need word splitting here to create the OS_STACKS array.
-# shellcheck disable=SC2207
-OS_STACKS=($(openstack stack list \
- -f value -c "Stack Name" -c "Stack Status" \
- --property "stack_status=CREATE_COMPLETE" \
- --property "stack_status=DELETE_FAILED" \
- --property "stack_status=CREATE_FAILED" \
- | awk '{print $1}'))
-
-echo "---> Active stacks"
-for stack in "${OS_STACKS[@]}"; do
- echo "$stack"
-done
-
-##########################
-## DELETE UNUSED STACKS ##
-##########################
-echo "---> Delete orphaned stacks"
-
-# Search for stacks that are not in use by either releng or sandbox silos and
-# delete them.
-for STACK_NAME in "${OS_STACKS[@]}"; do
- echo "Checking if orphaned $STACK_NAME"
-
- # JENKINS_URLS is provided by the Jenkins Job declaration and intentially
- # needs to be globbed.
- # shellcheck disable=SC2153,SC2086
- if stack_in_jenkins "$STACK_NAME" $JENKINS_URLS; then
- # No need to delete stacks if there exists an active build for them
- continue
- else
- status=$(openstack stack show -f value -c "stack_status" "$STACK_NAME")
- case "$status" in
- DELETE_IN_PROGRESS)
- echo "skipping delete, $STACK_NAME is already DELETE in progress."
- continue
- ;;
- DELETE_FAILED)
- # Abandon is not supported in Vexxhost so let's keep trying to
- # delete for now...
- # echo "Stack delete failed, trying to stack abandon now."
- # openstack stack abandon "$STACK_NAME"
- echo "Deleting orphaned stack: $STACK_NAME"
- openstack stack delete --yes "$STACK_NAME"
-
- echo "------------------------------------"
- echo "Stack details"
- echo "------------------------------------"
- openstack stack show "$STACK_NAME" -f yaml
- echo "------------------------------------"
- continue
- ;;
- CREATE_COMPLETE|CREATE_FAILED)
- echo "Deleting orphaned stack: $STACK_NAME"
- openstack stack delete --yes "$STACK_NAME"
- continue
- ;;
- *)
- continue
- ;;
- esac
- fi
-done
+++ /dev/null
-#!/bin/bash -l
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2018 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-##############################################################################
-# Scans OpenStack for orphaned volumes
-
-mapfile -t os_volumes < <(openstack volume list -f value -c ID --status Available)
-
-echo "---> Orphaned volumes"
-if [ ${#os_volumes[@]} -eq 0 ]; then
- echo "No orphaned volumes found."
-else
- for volume in "${os_volumes[@]}"; do
- echo "Removing volume $volume"
- lftools openstack --os-cloud vex volume remove --minutes 15 "$volume"
- done
-fi
- gerrit-jjb-verify
- builder-check-poms
- builder-copy-sandbox-logs
- - builder-openstack-cron
- gerrit-tox-verify
# Automation for docs and jobs
- 'builder-update-image-list'
https://jenkins.opendaylight.org/releng
https://jenkins.opendaylight.org/sandbox
+- project:
+ name: builder-openstack
+ jobs:
+ - gerrit-openstack-cron
+
+ project: 'releng/builder'
+ project-name: builder
+ build-node: centos7-builder-2c-2g
+
+ jenkins-urls: >
+ https://jenkins.opendaylight.org
+ https://jenkins.opendaylight.org/sandbox
+ openstack-cloud: vex
+
- job-template:
name: builder-check-poms
node: centos7-builder-2c-1g
- lf-infra-publish
-- job-template:
- name: builder-openstack-cron
- project-type: freestyle
- node: '{build-node}'
- concurrent: true
-
- properties:
- - lf-infra-properties:
- build-days-to-keep: 7
-
- parameters:
- - lf-infra-parameters:
- project: '{project}'
- stream: '{stream}'
- branch: '{branch}'
- lftools-version: '{lftools-version}'
- - string:
- name: OS_CLOUD
- default: '{os-cloud}'
- description: |
- The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
- variable name that is significant to openstack client as a
- environment variable. Please refer to the documentation for
- further details.
- https://docs.openstack.org/developer/python-openstackclient/
- - string:
- name: ARCHIVE_ARTIFACTS
- default: '{archive-artifacts}'
- description: Artifacts to archive to the logs server.
- - string:
- name: JENKINS_URLS
- default: '{jenkins-urls}'
- description: 'Space separated list of Jenkins URLs to check for active builds'
-
- scm:
- - git-scm:
- branch: '{branch}'
-
- wrappers:
- - lf-infra-wrappers:
- build-timeout: '{build-timeout}'
- jenkins-ssh-credential: '{jenkins-ssh-credential}'
- # Listed after to override openstack-infra-wrappers clouds.yaml definition
- - config-file-provider:
- files:
- - file-id: clouds-yaml
- target: '$HOME/.config/openstack/clouds.yaml'
- - file-id: npmrc
- target: '$HOME/.npmrc'
- - file-id: pipconf
- target: '$HOME/.config/pip/pip.conf'
-
- triggers:
- - timed: '0,30 * * * *'
-
- builders:
- - lf-infra-pre-build
- - shell: |
- #!/bin/bash -l
- pip install --user --upgrade lftools[openstack]~=0.17.1
- # Servers
- - odl-openstack-cleanup-stale-stacks
- - odl-openstack-cleanup-stale-nodes
- - odl-openstack-cleanup-orphaned-nodes
- # Volumes
- - odl-openstack-cleanup-stale-volumes
- # Images
- - odl-openstack-check-image-protection
- - odl-openstack-cleanup-old-images
-
- publishers:
- - lf-infra-publish
-
- job-template:
name: 'builder-update-image-list'
project-type: freestyle
send-to:
- recipients
-- builder:
- name: odl-openstack-check-image-protection
- builders:
- - shell: !include-raw: odl-openstack-check-image-protection.sh
-
-- builder:
- name: odl-openstack-cleanup-old-images
- builders:
- - shell: !include-raw: odl-openstack-cleanup-old-images.sh
-
-- builder:
- name: odl-openstack-cleanup-orphaned-nodes
- builders:
- - shell: !include-raw: odl-openstack-cleanup-orphaned-nodes.sh
-
-- builder:
- name: odl-openstack-cleanup-stale-nodes
- builders:
- - shell: !include-raw: odl-openstack-cleanup-stale-nodes.sh
-
-- builder:
- name: odl-openstack-cleanup-stale-stacks
- builders:
- - shell: !include-raw: odl-openstack-cleanup-stale-stacks.sh
-
-- builder:
- name: odl-openstack-cleanup-stale-volumes
- builders:
- - shell: !include-raw: odl-openstack-cleanup-stale-volumes.sh
-
- builder:
name: opendaylight-infra-stack
# opendaylight-infra-stack.sh has a required variable {stack-template} that
Code Review / releng / builder.git / commitdiff