+++ /dev/null
-#!/usr/bin/awk -f
-# SPDX-License-Identifier: EPL-1.0
-##############################################################################
-# Copyright (c) 2017 The Linux Foundation and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-#
-##############################################################################
-
-BEGIN {
- new_tag = new_reltag # new release tag
- curr_tag = curr_reltag # current release tag
- prev_tag = prev_reltag # previous release tag
-
- new_release = tolower(new_tag)
- curr_release = tolower(curr_tag)
- prev_release = tolower(prev_tag)
-
- ws = "[\\t ]*" # white-spaces
- startpat = "^" ws "- project:" # start pattern
- endpat = startpat # end pattern
- op = "^" ws "---" ws "$" # match files starts with "---"
-
- next_release_tag = "^" ws "next-release-tag: '{stream}'"
- master = "'master'"
- new_branch = "'stable/" new_release "'"
- curr_branch = "'stable/" curr_release "'"
- prev_branch = "'stable/" prev_release "'"
-
- # replace block to add new release
- new_rel_yaml_tag = "- " new_release ":";
- br_master_yaml_tag = " branch: 'master'";
- jre_yaml_tag = " jre: 'openjdk11'";
- java_version_yaml_tag = " java-version: 'openjdk11'";
- curr_rel_yaml_tag = "- " curr_release ":";
- br_stable_curr_yaml_tag = " branch: 'stable/" curr_release "'";
-
- # replace block for autorelease-projects
- #new_rel_yaml_tag = "- " new_release ":";
- next_rel_tag_new_yaml_tag = " next-release-tag: '{stream}'";
- #br_master_yaml_tag = " branch: 'master'";
- jdk_yaml_tag = " jdk: 'openjdk8'";
- intg_test_yaml_tag = " integration-test: " new_release;
- extra_mvn_opts_tag = " extra-mvn-opts: -Dsft.heap.max=4g"
- #curr_rel_yaml_tag = "- " curr_release ":";
- next_rel_tag_curr_yaml_tag = " next-release-tag: '{stream}'";
- #br_stable_curr_yaml_tag = " branch: 'stable/" curr_release "'";
-
- # search patterns
- smaster = "^" ws "- master:"
- sstream = "^" ws "stream:"
- srelease = "^" ws "- " curr_release ":"
- snext_release_tag = "^" ws "next-release-tag:"
- #if (l ~ next_release_tag) { next_release_tag = 1; continue; }
- sbranch = "^" ws "branch: " master
- sfunctionality = "^" ws "functionality:"
-
- debug = 0 # set to 1 to print debug info
- file_format = 2 # project stream format
-
- release_found = 0
- stream_found = 0
- nrt_found = 0
- func_found = 0
-}
-
-{
- # exit if release info is not available
- if ((length(new_release) == 0 || length(curr_release) == 0)) {
- exit;
- }
-
- # read all lines of the file into an array
- file[NR] = $0
-}
-
-END {
- n = NR # total number of lines
- find_blks() # gets number of blocks between start/end pattern
- process_blk(arr_bs[1], arr_be[1], 1) # pass start and end of each block and process first block
- update_file(arr_be[1]) # write processed content
-
- if (debug) {
- print "number of blocks="nb;
- print "total records in file[]="length(f);
- print "size of firstblk[]="length(firstblk);
- print "size of newblk[]="length(newblk);
- print "size of oldmaster[]="length(oldmaster);
- print "size of newblk[]="length(newblk);
- }
-}
-
-function find_blks( i, l, bs, be) {
- for (i = 1; i <= n; i++) {
- l = file[i]
- if (l ~ startpat) project = 1 # start pattern
- if (bs > be && l ~ endpat) arr_be[++be] = i - 1 # block end
- if ( l ~ startpat) arr_bs[++bs] = i - 1 # block start
- }
- nb = be
-
- # to handle files with single blocks
- if (nb == 0 && length(file) > 1 && project == 1) {
- nb = 1
- arr_bs[1] = 1 # start after line '---'
- arr_be[1] = length(file) # set length of the file
- }
-
- if (debug) {
- for (i = 1; i < nb; i++)
- print "find_blks: nb=" nb " arr_bs[" i "]="arr_bs[i]" arr_be[" i "]="arr_be[i];
- }
-}
-
-function process_blk(bs, be, bn, i, l) {
- if (debug) {
- print "process_blk: bn=" bn ", bs=" bs " ,be=" be
- }
-
- # get the first block
- for (i = bs + 1; i <= be ; i++) {
- l = file[i]
- # determine file format
- if (l ~ /stream:/) {
- x=index(l,":")
- s = substr(l, x+2, length(l) - x)
- if (s == curr_release || s == new_release) {
- file_format = 1
- } else if (length(s) == 0 ) {
- file_format = 0
- }
- }
- firstblk[++nex] = l
- }
-
- if (debug) {
- print "process_blk: stream='" s "' length(s)=" length(s)" file_format='" file_format "'"
- }
-
- # Handle single stream format
- if (file_format == 1) {
- # create new block to be inserted
- for (i = 1; i <= length(firstblk); i++) {
- l = firstblk[i]
- if (l ~ /name:|stream:/) sub(curr_release, new_release, l)
- newblk[++nex1] = l
- }
- # re-create old block and change master to stable/branch
- for (i = 1; i <= length(firstblk)-1; i++) {
- l = firstblk[i]
- if (l ~ /branch:/) sub(master, curr_branch, l)
- oldmaster[++nex2] = l
- }
- } else if (file_format == 0) {
- # Handle multi-stream format
- for (i = 1; i <= length(firstblk)-1; i++) {
- l = firstblk[i]
- if (l ~ sstream) { stream_found = 1; }
- if (l ~ srelease) { release_found = 1; indent = substr(l, 1, index(l, "-")-1); continue; }
- if (l ~ sfunctionality) { func_found = 1; }
- if (l ~ snext_release_tag) { nrt_found = 1; }
- if (l ~ sbranch) {
- # append lines
- if (stream_found && release_found && !nrt_found) {
- newblk[++nex3] = indent new_rel_yaml_tag;
- newblk[++nex3] = indent br_master_yaml_tag;
- newblk[++nex3] = indent java_version_yaml_tag;
- newblk[++nex3] = indent curr_rel_yaml_tag;
- newblk[++nex3] = indent br_stable_curr_yaml_tag;
- stream_found = 0;
- release_found = 0;
- func_found = 0;
- continue;
- }
- if (stream_found && release_found && nrt_found) {
- newblk[++nex3] = indent new_rel_yaml_tag;
- newblk[++nex3] = indent next_rel_tag_new_yaml_tag;
- newblk[++nex3] = indent br_master_yaml_tag;
- newblk[++nex3] = indent intg_test_yaml_tag;
- newblk[++nex3] = indent extra_mvn_opts_tag;
- newblk[++nex3] = indent curr_rel_yaml_tag;
- newblk[++nex3] = indent next_rel_tag_curr_yaml_tag;
- newblk[++nex3] = indent br_stable_curr_yaml_tag;
- stream_found = 0; release_found = 0; nrt_found=0;
- continue;
- }
- }
- newblk[++nex3] = l
-
- if (debug) {
- print "process_blk: append(newblk[]) : stream="stream" release_found="release_found
- }
- }
- } else {
- # exit on unknown file format
- exit;
- }
-}
-
-function update_file(be, i, j, l) {
- i = 1
- # handle lines before "---"
- while (i <= n) {
- print l = file[i++]
- if (l ~ op) break
- }
-
- if (debug) {
- print "writing master block"
- }
-
- # Handle single stream format
- if (file_format == 1) {
- for (j = 1; j <= nex1; j++) # write new branch block
- print newblk[j]
-
- if (debug) {
- print "writing stable block"
- }
-
- for (j = 1; j <= nex2; j++) # write updated branch block
- print oldmaster[j]
-
- # Handle multi-stream format
- } else if (file_format == 0) {
- # print the first block
- for (j = 1; j <= nex3; j++) # write multi-stream block
- print newblk[j]
- }
-
- if (debug) {
- print "writing rest of the file"
- }
-
- while (be <= n) { # write rest of the file
- print file[be++]
- }
-}
--- /dev/null
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2020 Thanh Ha
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+#
+##############################################################################
+"""Script for cutting new jobs when branching a new stable release."""
+
+import argparse
+from argparse import RawTextHelpFormatter
+import copy
+import fileinput
+import os
+import shutil
+import sys
+
+try:
+ import ruamel.yaml
+except ModuleNotFoundError:
+ print("ERROR: This script requires the package 'ruamel.yaml', please install it.")
+ print(
+ "If ruamel.yaml is not available in your system's package manager you"
+ " can install from PyPi with:"
+ )
+ print("")
+ print(" pip install --user ruamel.yaml")
+ sys.exit(1)
+
+yaml = ruamel.yaml.YAML()
+yaml.allow_duplicate_keys = True
+yaml.preserve_quotes = True
+
+default_branch = "master" # This is the primary dev branch of the project
+
+
+def create_and_update_project_jobs(
+ release_on_stable_branch, release_on_current_branch, job_dir
+):
+ """Create and update project build jobs for the current and next dev release.
+
+ Project jobs are jobs defined in the project.yaml that have the same name
+ the directory they are in.
+
+ Only updates projects where the top project configuration has a name that
+ is equivalent to the current release. For example project name
+ "aaa-silicon" would have a release that matches what was passed to
+ release_on_stable_branch.
+ """
+ for directory in filter(
+ lambda x: os.path.isdir(os.path.join(job_dir, x)), os.listdir(job_dir)
+ ):
+ try:
+ with open(
+ os.path.join(job_dir, directory, "{}.yaml".format(directory)), "r"
+ ) as f:
+ data = yaml.load(f)
+
+ # Only create new jobs if the top level project name matches
+ # release_on_stable_branch variable
+ if not data[0]["project"]["name"] == "{}-{}".format(
+ directory, release_on_stable_branch
+ ):
+ continue
+
+ # Create a new job for the next release on the default_branch
+ new_job = copy.deepcopy(data[0])
+ new_job["project"]["name"] = "{}-{}".format(
+ directory, release_on_current_branch
+ )
+ new_job["project"]["branch"] = default_branch
+ new_job["project"]["stream"] = "{}".format(release_on_current_branch)
+
+ # Update exiting job for the new stable branch
+ data[0]["project"]["branch"] = "stable/{}".format(
+ release_on_stable_branch
+ )
+
+ data.insert(0, new_job)
+
+ with open(
+ os.path.join(job_dir, directory, "{}.yaml".format(directory)), "w"
+ ) as f:
+ stream = ruamel.yaml.round_trip_dump(data)
+ f.write("---\n")
+ f.write(stream)
+ except FileNotFoundError: # If project.yaml file does not exist we can skip
+ pass
+
+
+def update_job_streams(release_on_stable_branch, release_on_current_branch, job_dir):
+ """Update projects that have a stream variable that is a list.
+
+ If a stream variable is a list that means the project likely has multiple
+ maintainance branches supported.
+
+ This function also does not support {project}.yaml files as parsing those
+ are handled by other functions in this script.
+
+ Only updates projects where the top stream in the list is equivalent to the
+ current release. For example stream "silicon" would have a release that
+ matches what was passed to release_on_stable_branch.
+ """
+ for directory in filter(
+ lambda d: os.path.isdir(os.path.join(job_dir, d)), os.listdir(job_dir)
+ ):
+ for job_file in filter(
+ lambda f: os.path.isfile(os.path.join(job_dir, directory, f)),
+ os.listdir(os.path.join(job_dir, directory)),
+ ):
+
+ # Projects may have non-yaml files in their repos so ignore them.
+ if not job_file.endswith(".yaml"):
+ continue
+
+ # Ignore project.yaml files as they are not supported by this function.
+ if job_file == "{}.yaml".format(directory):
+ continue
+
+ file_changed = False
+
+ with open(os.path.join(job_dir, directory, job_file), "r") as f:
+ data = yaml.load(f)
+
+ for project in data:
+ streams = project.get("project", {}).get("stream", None)
+
+ if not isinstance(streams, list): # We only support lists streams
+ continue
+
+ # Skip if the stream does not match
+ # release_on_stable_branch in the first item
+ if not streams[0].get(release_on_stable_branch, None):
+ continue
+
+ # Create the next release stream
+ new_stream = {}
+ new_stream[release_on_current_branch] = copy.deepcopy(
+ streams[0].get(release_on_stable_branch)
+ )
+
+ # Update the previous release stream branch to
+ # stable/{stream} instead of default_branch
+ streams[0][release_on_stable_branch]["branch"] = "stable/{}".format(
+ release_on_stable_branch
+ )
+
+ streams.insert(0, new_stream)
+ file_changed = True
+
+ # Because we are looping every file we only want to save if we made changes.
+ if file_changed:
+ with open(os.path.join(job_dir, directory, job_file), "w") as f:
+ stream = ruamel.yaml.round_trip_dump(data)
+ f.write("---\n")
+ f.write(stream)
+
+
+def update_integration_csit_list(
+ release_on_stable_branch, release_on_current_branch, job_dir
+):
+ """Update csit-*-list variables and files integration-test-jobs.yaml."""
+
+ class Generic:
+ def __init__(self, tag, value, style=None):
+ self._value = value
+ self._tag = tag
+ self._style = style
+
+ class GenericScalar(Generic):
+ @classmethod
+ def to_yaml(self, representer, node):
+ return representer.represent_scalar(node._tag, node._value)
+
+ @staticmethod
+ def construct(constructor, node):
+ return constructor.construct_scalar(node)
+
+ def default_constructor(constructor, tag_suffix, node):
+ generic = {ruamel.yaml.ScalarNode: GenericScalar,}.get( # noqa
+ type(node)
+ )
+ if generic is None:
+ raise NotImplementedError("Node: " + str(type(node)))
+ style = getattr(node, "style", None)
+ instance = generic.__new__(generic)
+ yield instance
+ state = generic.construct(constructor, node)
+ instance.__init__(tag_suffix, state, style=style)
+
+ ruamel.yaml.add_multi_constructor(
+ "", default_constructor, Loader=ruamel.yaml.SafeLoader
+ )
+ yaml.register_class(GenericScalar)
+
+ integration_test_jobs_yaml = os.path.join(
+ job_dir, "integration", "integration-test-jobs.yaml"
+ )
+
+ with open(integration_test_jobs_yaml, "r") as f:
+ data = yaml.load(f)
+
+ for project in data:
+ # Skip items that are not of "project" type
+ if not project.get("project"):
+ continue
+
+ streams = project.get("project", {}).get("stream", None)
+
+ # Skip projects that do not have a stream configured
+ if not isinstance(streams, list): # We only support lists streams
+ continue
+
+ # Skip if the stream does not match
+ # release_on_current_branch in the first item
+ if not streams[0].get(release_on_current_branch, None):
+ continue
+
+ # Update csit-list parameters for next release
+ if streams[0][release_on_current_branch].get("csit-list"):
+ update_stream = streams[0][release_on_current_branch]
+ update_stream["csit-list"] = GenericScalar(
+ "!include:", "csit-jobs-{}.lst".format(release_on_current_branch)
+ )
+
+ # Update csit-mri-list parameters for next release
+ if streams[0][release_on_current_branch].get("csit-mri-list"):
+ update_stream = streams[0][release_on_current_branch]
+ update_stream["csit-mri-list"] = "{{csit-mri-list-{}}}".format(
+ release_on_current_branch
+ )
+
+ # Update csit-weekly-list parameters for next release
+ if streams[0][release_on_current_branch].get("csit-weekly-list"):
+ update_stream = streams[0][release_on_current_branch]
+ update_stream["csit-weekly-list"] = "{{csit-weekly-list-{}}}".format(
+ release_on_current_branch
+ )
+
+ # Update csit-sanity-list parameters for next release
+ if streams[0][release_on_current_branch].get("csit-sanity-list"):
+ update_stream = streams[0][release_on_current_branch]
+ update_stream["csit-sanity-list"] = "{{csit-sanity-list-{}}}".format(
+ release_on_current_branch
+ )
+
+ with open(integration_test_jobs_yaml, "w") as f:
+ stream = ruamel.yaml.round_trip_dump(data)
+ f.write("---\n")
+ f.write(stream)
+
+ # Update the csit-*-list variables in defaults.yaml
+
+ defaults_yaml = os.path.join(job_dir, "defaults.yaml")
+
+ with open(defaults_yaml, "r") as f:
+ data = yaml.load(f)
+
+ # Add next release csit-mri-list-RELEASE
+ new_csit_mri_list = copy.deepcopy(
+ data[0]["defaults"].get("csit-mri-list-{}".format(release_on_stable_branch))
+ )
+ data[0]["defaults"][
+ "csit-mri-list-{}".format(release_on_current_branch)
+ ] = new_csit_mri_list.replace(
+ release_on_stable_branch, release_on_current_branch
+ )
+
+ # Add next release csit-mri-list-RELEASE
+ new_csit_mri_list = copy.deepcopy(
+ data[0]["defaults"].get("csit-mri-list-{}".format(release_on_stable_branch))
+ )
+ data[0]["defaults"][
+ "csit-mri-list-{}".format(release_on_current_branch)
+ ] = new_csit_mri_list.replace(
+ release_on_stable_branch, release_on_current_branch
+ )
+
+ # Add next release csit-weekly-list-RELEASE
+ new_csit_mri_list = copy.deepcopy(
+ data[0]["defaults"].get(
+ "csit-weekly-list-{}".format(release_on_stable_branch)
+ )
+ )
+ data[0]["defaults"][
+ "csit-weekly-list-{}".format(release_on_current_branch)
+ ] = new_csit_mri_list.replace(
+ release_on_stable_branch, release_on_current_branch
+ )
+
+ # Add next release csit-sanity-list-RELEASE
+ new_csit_mri_list = copy.deepcopy(
+ data[0]["defaults"].get(
+ "csit-sanity-list-{}".format(release_on_stable_branch)
+ )
+ )
+ data[0]["defaults"][
+ "csit-sanity-list-{}".format(release_on_current_branch)
+ ] = new_csit_mri_list.replace(
+ release_on_stable_branch, release_on_current_branch
+ )
+
+ with open(defaults_yaml, "w") as f:
+ stream = ruamel.yaml.round_trip_dump(data)
+ f.write("---\n")
+ f.write(stream)
+
+ # Handle copying and updating the csit-*.lst files
+ csit_file = "csit-jobs-{}.lst".format(release_on_stable_branch)
+ src = os.path.join(job_dir, "integration", csit_file)
+ dest = os.path.join(
+ job_dir,
+ "integration",
+ csit_file.replace(release_on_stable_branch, release_on_current_branch),
+ )
+ shutil.copyfile(src, dest)
+ with fileinput.FileInput(dest, inplace=True) as file:
+ for line in file:
+ print(
+ line.replace(release_on_stable_branch, release_on_current_branch),
+ end="",
+ )
+
+
+parser = argparse.ArgumentParser(
+ description="""Creates & updates jobs for ODL projects when branch cutting.
+
+ Example usage: python scripts/cut-branch.sh Silicon Phosphorus jjb/
+
+ ** If calling from tox the JOD_DIR is auto-detected so only pass the current
+ and next release stream name. **
+ """,
+ formatter_class=RawTextHelpFormatter,
+)
+parser.add_argument(
+ "release_on_stable_branch",
+ metavar="RELEASE_ON_STABLE_BRANCH",
+ type=str,
+ help="The ODL release codename for the stable branch that was cut.",
+)
+parser.add_argument(
+ "release_on_current_branch",
+ metavar="RELEASE_ON_CURRENT_BRANCH",
+ type=str,
+ help="""The ODL release codename for the new {}
+ (eg. Magnesium, Aluminium, Silicon).""".format(
+ default_branch
+ ),
+)
+parser.add_argument(
+ "job_dir",
+ metavar="JOB_DIR",
+ type=str,
+ help="Path to the directory containing JJB config.",
+)
+args = parser.parse_args()
+
+# We only handle lower release codenames
+release_on_stable_branch = args.release_on_stable_branch.lower()
+release_on_current_branch = args.release_on_current_branch.lower()
+
+create_and_update_project_jobs(
+ release_on_stable_branch, release_on_current_branch, args.job_dir
+)
+update_job_streams(release_on_stable_branch, release_on_current_branch, args.job_dir)
+update_integration_csit_list(
+ release_on_stable_branch, release_on_current_branch, args.job_dir
+)
Code Review / releng / builder.git / commitdiff