Add job to auto-update new built packer images

The job auto-updates packer images to the latest once they are built
every month into the releng/builder repository and push a commit
to Gerrit. The CR can be merged and reduces the wait time when we
want images to be updated.

Steps followed:
1. Get a list of images from the releng/builder repository
2. Search openstack cloud for the latest images available
3. Compare the time stamps of the new image with the image in use
4. Update the image to the config files and yaml files
5. Push the change to Gerrit

Change-Id: I592219df0ad5eb916448b4c905931b354f407611
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>

diff --git a/jjb/opendaylight-infra-update-images.sh b/jjb/opendaylight-infra-update-images.sh
new file mode 100644 (file)
index 0000000..c77b696
--- /dev/null
+++ b/jjb/opendaylight-infra-update-images.sh
@@ -0,0 +1,61 @@
+#!/bin/bash -l
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# Auto-update packer images:
+# 1. Get a list of images from the releng/builder repository
+# 2. Search openstack cloud for the latest images available
+# 3. Compare the time stamps of the new image with the image in use
+# 4. Update the image to the config files and yaml files
+# 5. Push the change to Gerrit
+
+virtualenv "/tmp/v/openstack"
+# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091
+source "/tmp/v/openstack/bin/activate"
+pip install --upgrade --quiet "pip<10.0.0" setuptools
+pip install --upgrade --quiet python-openstackclient
+pip freeze
+
+mkdir -p "$WORKSPACE/archives"
+echo "List of images used on the source repository:"
+grep -Er '(_system_image:|IMAGE_NAME)' \
+| grep  ZZCI | awk -F: -e '{print $3}' \
+| grep '\S' | tr -d \'\" | sort -n | uniq \
+| tee "$WORKSPACE/archives/used_image_list.txt"
+
+set -x
+
+while read -r line ; do
+    # remove leading white spaces if they exists
+    image_in_use="${line#"${line%%[![:space:]]*}"}"
+    # remove trailing white spaces if they exists
+    image_in_use="${image_in_use%"${image_in_use##*[![:space:]]}"}"
+    # get image type - ex: builder, docker, gbp etc
+    image_type="${line% -*}"
+    # get the latest image available on the cloud
+    new_image=$(openstack image list --long -f value -c Name -c Protected \
+        | grep "${image_type}.*False" | tail -n-1 | sed 's/ False//')
+    [[ -z ${new_image} ]] && continue
+    # strip the timestamp from the image name amd compare
+    new_image_isotime=${new_image##*- }
+    image_in_use_isotime=${image_in_use##*- }
+    # compare timestamps
+    if [[ ${new_image_isotime//[\-\.]/} -gt ${image_in_use_isotime//[\-\.]/} ]]; then
+        # generate a patch to be submited to Gerrit
+        echo "Update old image: ${image_in_use} with new image: ${new_image}"
+        grep -rlE '(_system_image:|IMAGE_NAME)' | xargs sed -i "s/${image_in_use}/${new_image}/"
+    else
+        echo "No new image to update: ${new_image}"
+    fi
+done < "$WORKSPACE/archives/used_image_list.txt"
+
+git remote -v
+git add -u
+git status

diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index 69d329963a08fb169042b7831bd07263f978167d..15005dc954967ee55794ec4958feab43b5160eaf 100644 (file)
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -22,6 +22,7 @@
       - gerrit-tox-verify
       # Automation for docs and jobs
       - 'builder-update-image-list'
+      - 'builder-update-images'
 
     # The following values are only specified here
     # so that code blocks look similar to other projects.
@@ -173,3 +174,55 @@
           email-recipients: '{email-recipients}'
           email-prefix: '[releng]'
       - lf-infra-publish
+
+
+- job-template:
+    name: 'builder-update-images'
+    project-type: freestyle
+    node: centos7-builder-2c-1g
+
+    properties:
+      - opendaylight-infra-properties:
+          build-days-to-keep: 7
+
+    parameters:
+      - opendaylight-infra-parameters:
+          os-cloud: vex
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+          artifacts: '{archive-artifacts}'
+
+    scm:
+      - git-scm:
+          branch: '{branch}'
+
+    wrappers:
+      - opendaylight-infra-wrappers:
+          build-timeout: '{build-timeout}'
+      # Listed after to override openstack-infra-wrappers clouds.yaml definition
+      - config-file-provider:
+          files:
+            - file-id: clouds-yaml
+              target: '$HOME/.config/openstack/clouds.yaml'
+
+    triggers:
+      # Update images the day after packer images are built every month on 2nd
+      - timed: '00 H 2 * *'
+
+    builders:
+      - lf-infra-pre-build
+      - shell: !include-raw-escape: opendaylight-infra-update-images.sh
+      - lf-infra-push-gerrit-patch:
+          project: '{project}'
+          gerrit-user: 'jenkins-$SILO'
+          gerrit-host: 'git.opendaylight.org'
+          gerrit-topic: 'releng-update-cloud-images'
+          gerrit-commit-message: 'Update image to the latest'
+          reviewers-email: 'jenkins-releng@opendaylight.org'
+
+    publishers:
+      - email-notification:
+          email-recipients: '{email-recipients}'
+          email-prefix: '[releng]'
+      - lf-infra-publish