From 4577896e9dc18d02877a487602244e42625fc1d5 Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Thu, 5 Sep 2019 00:20:30 +0530 Subject: [PATCH] Add job to auto-update new built packer images The job auto-updates packer images to the latest once they are built every month into the releng/builder repository and push a commit to Gerrit. The CR can be merged and reduces the wait time when we want images to be updated. Steps followed: 1. Get a list of images from the releng/builder repository 2. Search openstack cloud for the latest images available 3. Compare the time stamps of the new image with the image in use 4. Update the image to the config files and yaml files 5. Push the change to Gerrit Change-Id: I592219df0ad5eb916448b4c905931b354f407611 Signed-off-by: Anil Belur --- jjb/opendaylight-infra-update-images.sh | 61 +++++++++++++++++++++++++ jjb/releng-jobs.yaml | 53 +++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 jjb/opendaylight-infra-update-images.sh diff --git a/jjb/opendaylight-infra-update-images.sh b/jjb/opendaylight-infra-update-images.sh new file mode 100644 index 000000000..c77b696b6 --- /dev/null +++ b/jjb/opendaylight-infra-update-images.sh @@ -0,0 +1,61 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Auto-update packer images: +# 1. Get a list of images from the releng/builder repository +# 2. Search openstack cloud for the latest images available +# 3. Compare the time stamps of the new image with the image in use +# 4. Update the image to the config files and yaml files +# 5. Push the change to Gerrit + +virtualenv "/tmp/v/openstack" +# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091 +source "/tmp/v/openstack/bin/activate" +pip install --upgrade --quiet "pip<10.0.0" setuptools +pip install --upgrade --quiet python-openstackclient +pip freeze + +mkdir -p "$WORKSPACE/archives" +echo "List of images used on the source repository:" +grep -Er '(_system_image:|IMAGE_NAME)' \ +| grep ZZCI | awk -F: -e '{print $3}' \ +| grep '\S' | tr -d \'\" | sort -n | uniq \ +| tee "$WORKSPACE/archives/used_image_list.txt" + +set -x + +while read -r line ; do + # remove leading white spaces if they exists + image_in_use="${line#"${line%%[![:space:]]*}"}" + # remove trailing white spaces if they exists + image_in_use="${image_in_use%"${image_in_use##*[![:space:]]}"}" + # get image type - ex: builder, docker, gbp etc + image_type="${line% -*}" + # get the latest image available on the cloud + new_image=$(openstack image list --long -f value -c Name -c Protected \ + | grep "${image_type}.*False" | tail -n-1 | sed 's/ False//') + [[ -z ${new_image} ]] && continue + # strip the timestamp from the image name amd compare + new_image_isotime=${new_image##*- } + image_in_use_isotime=${image_in_use##*- } + # compare timestamps + if [[ ${new_image_isotime//[\-\.]/} -gt ${image_in_use_isotime//[\-\.]/} ]]; then + # generate a patch to be submited to Gerrit + echo "Update old image: ${image_in_use} with new image: ${new_image}" + grep -rlE '(_system_image:|IMAGE_NAME)' | xargs sed -i "s/${image_in_use}/${new_image}/" + else + echo "No new image to update: ${new_image}" + fi +done < "$WORKSPACE/archives/used_image_list.txt" + +git remote -v +git add -u +git status diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml index 69d329963..15005dc95 100644 --- a/jjb/releng-jobs.yaml +++ b/jjb/releng-jobs.yaml @@ -22,6 +22,7 @@ - gerrit-tox-verify # Automation for docs and jobs - 'builder-update-image-list' + - 'builder-update-images' # The following values are only specified here # so that code blocks look similar to other projects. @@ -173,3 +174,55 @@ email-recipients: '{email-recipients}' email-prefix: '[releng]' - lf-infra-publish + + +- job-template: + name: 'builder-update-images' + project-type: freestyle + node: centos7-builder-2c-1g + + properties: + - opendaylight-infra-properties: + build-days-to-keep: 7 + + parameters: + - opendaylight-infra-parameters: + os-cloud: vex + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + artifacts: '{archive-artifacts}' + + scm: + - git-scm: + branch: '{branch}' + + wrappers: + - opendaylight-infra-wrappers: + build-timeout: '{build-timeout}' + # Listed after to override openstack-infra-wrappers clouds.yaml definition + - config-file-provider: + files: + - file-id: clouds-yaml + target: '$HOME/.config/openstack/clouds.yaml' + + triggers: + # Update images the day after packer images are built every month on 2nd + - timed: '00 H 2 * *' + + builders: + - lf-infra-pre-build + - shell: !include-raw-escape: opendaylight-infra-update-images.sh + - lf-infra-push-gerrit-patch: + project: '{project}' + gerrit-user: 'jenkins-$SILO' + gerrit-host: 'git.opendaylight.org' + gerrit-topic: 'releng-update-cloud-images' + gerrit-commit-message: 'Update image to the latest' + reviewers-email: 'jenkins-releng@opendaylight.org' + + publishers: + - email-notification: + email-recipients: '{email-recipients}' + email-prefix: '[releng]' + - lf-infra-publish -- 2.36.6