scripts/cut-branch-jobs.py

   1 # SPDX-License-Identifier: EPL-1.0
   2 ##############################################################################
   3 # Copyright (c) 2020 Thanh Ha
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Eclipse Public License v1.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.eclipse.org/legal/epl-v10.html
   9 #
  10 ##############################################################################
  11 """Script for cutting new jobs when branching a new stable release."""
  12
  13 import argparse
  14 from argparse import RawTextHelpFormatter
  15 import copy
  16 import fileinput
  17 import os
  18 import shutil
  19 import sys
  20
  21 try:
  22     import ruamel.yaml
  23 except ModuleNotFoundError:
  24     print("ERROR: This script requires the package 'ruamel.yaml', please install it.")
  25     print(
  26         "If ruamel.yaml is not available in your system's package manager you"
  27         " can install from PyPi with:"
  28     )
  29     print("")
  30     print("    pip install --user ruamel.yaml")
  31     sys.exit(1)
  32
  33 yaml = ruamel.yaml.YAML()
  34 yaml.allow_duplicate_keys = True
  35 yaml.preserve_quotes = True
  36
  37 default_branch = "master"  # This is the primary dev branch of the project
  38
  39
  40 def create_and_update_project_jobs(
  41     release_on_stable_branch, release_on_current_branch, job_dir
  42 ):
  43     """Create and update project build jobs for the current and next dev release.
  44
  45     Project jobs are jobs defined in the project.yaml that have the same name
  46     the directory they are in.
  47
  48     Only updates projects where the top project configuration has a name that
  49     is equivalent to the current release. For example project name
  50     "aaa-silicon" would have a release that matches what was passed to
  51     release_on_stable_branch.
  52     """
  53     for directory in filter(
  54         lambda x: os.path.isdir(os.path.join(job_dir, x)), os.listdir(job_dir)
  55     ):
  56         try:
  57             with open(
  58                 os.path.join(job_dir, directory, "{}.yaml".format(directory)), "r"
  59             ) as f:
  60                 data = yaml.load(f)
  61
  62                 # Only create new jobs if the top level project name matches
  63                 # release_on_stable_branch variable
  64                 if not data[0]["project"]["name"] == "{}-{}".format(
  65                     directory, release_on_stable_branch
  66                 ):
  67                     continue
  68
  69                 # Create a new job for the next release on the default_branch
  70                 new_job = copy.deepcopy(data[0])
  71                 new_job["project"]["name"] = "{}-{}".format(
  72                     directory, release_on_current_branch
  73                 )
  74                 new_job["project"]["branch"] = default_branch
  75                 new_job["project"]["stream"] = "{}".format(release_on_current_branch)
  76
  77                 # Update exiting job for the new stable branch
  78                 data[0]["project"]["branch"] = "stable/{}".format(
  79                     release_on_stable_branch
  80                 )
  81
  82                 data.insert(0, new_job)
  83
  84             with open(
  85                 os.path.join(job_dir, directory, "{}.yaml".format(directory)), "w"
  86             ) as f:
  87                 stream = ruamel.yaml.round_trip_dump(data)
  88                 f.write("---\n")
  89                 f.write(stream)
  90         except FileNotFoundError:  # If project.yaml file does not exist we can skip
  91             pass
  92
  93
  94 def update_job_streams(release_on_stable_branch, release_on_current_branch, job_dir):
  95     """Update projects that have a stream variable that is a list.
  96
  97     If a stream variable is a list that means the project likely has multiple
  98     maintainance branches supported.
  99
 100     This function also does not support {project}.yaml files as parsing those
 101     are handled by other functions in this script.
 102
 103     Only updates projects where the top stream in the list is equivalent to the
 104     current release. For example stream "silicon" would have a release that
 105     matches what was passed to release_on_stable_branch.
 106     """
 107     for directory in filter(
 108         lambda d: os.path.isdir(os.path.join(job_dir, d)), os.listdir(job_dir)
 109     ):
 110         for job_file in filter(
 111             lambda f: os.path.isfile(os.path.join(job_dir, directory, f)),
 112             os.listdir(os.path.join(job_dir, directory)),
 113         ):
 114
 115             # Projects may have non-yaml files in their repos so ignore them.
 116             if not job_file.endswith(".yaml"):
 117                 continue
 118
 119             # Ignore project.yaml files as they are not supported by this function.
 120             if job_file == "{}.yaml".format(directory):
 121                 continue
 122
 123             file_changed = False
 124
 125             with open(os.path.join(job_dir, directory, job_file), "r") as f:
 126                 data = yaml.load(f)
 127
 128                 for project in data:
 129                     streams = project.get("project", {}).get("stream", None)
 130
 131                     if not isinstance(streams, list):  # We only support lists streams
 132                         continue
 133
 134                     # Skip if the stream does not match
 135                     # release_on_stable_branch in the first item
 136                     if not streams[0].get(release_on_stable_branch, None):
 137                         continue
 138
 139                     # Create the next release stream
 140                     new_stream = {}
 141                     new_stream[release_on_current_branch] = copy.deepcopy(
 142                         streams[0].get(release_on_stable_branch)
 143                     )
 144
 145                     # Update the previous release stream branch to
 146                     # stable/{stream} instead of default_branch
 147                     streams[0][release_on_stable_branch]["branch"] = "stable/{}".format(
 148                         release_on_stable_branch
 149                     )
 150
 151                     streams.insert(0, new_stream)
 152                     file_changed = True
 153
 154             # Because we are looping every file we only want to save if we made changes.
 155             if file_changed:
 156                 with open(os.path.join(job_dir, directory, job_file), "w") as f:
 157                     stream = ruamel.yaml.round_trip_dump(data)
 158                     f.write("---\n")
 159                     f.write(stream)
 160
 161
 162 def update_integration_csit_list(
 163     release_on_stable_branch, release_on_current_branch, job_dir
 164 ):
 165     """Update csit-*-list variables and files integration-test-jobs.yaml."""
 166
 167     class Generic:
 168         def __init__(self, tag, value, style=None):
 169             self._value = value
 170             self._tag = tag
 171             self._style = style
 172
 173     class GenericScalar(Generic):
 174         @classmethod
 175         def to_yaml(self, representer, node):
 176             return representer.represent_scalar(node._tag, node._value)
 177
 178         @staticmethod
 179         def construct(constructor, node):
 180             return constructor.construct_scalar(node)
 181
 182     def default_constructor(constructor, tag_suffix, node):
 183         generic = {ruamel.yaml.ScalarNode: GenericScalar,}.get(  # noqa
 184             type(node)
 185         )
 186         if generic is None:
 187             raise NotImplementedError("Node: " + str(type(node)))
 188         style = getattr(node, "style", None)
 189         instance = generic.__new__(generic)
 190         yield instance
 191         state = generic.construct(constructor, node)
 192         instance.__init__(tag_suffix, state, style=style)
 193
 194     ruamel.yaml.add_multi_constructor(
 195         "", default_constructor, Loader=ruamel.yaml.SafeLoader
 196     )
 197     yaml.register_class(GenericScalar)
 198
 199     integration_test_jobs_yaml = os.path.join(
 200         job_dir, "integration", "integration-test-jobs.yaml"
 201     )
 202
 203     with open(integration_test_jobs_yaml, "r") as f:
 204         data = yaml.load(f)
 205
 206         for project in data:
 207             # Skip items that are not of "project" type
 208             if not project.get("project"):
 209                 continue
 210
 211             streams = project.get("project", {}).get("stream", None)
 212
 213             # Skip projects that do not have a stream configured
 214             if not isinstance(streams, list):  # We only support lists streams
 215                 continue
 216
 217             # Skip if the stream does not match
 218             # release_on_current_branch in the first item
 219             if not streams[0].get(release_on_current_branch, None):
 220                 continue
 221
 222             # Update csit-list parameters for next release
 223             if streams[0][release_on_current_branch].get("csit-list"):
 224                 update_stream = streams[0][release_on_current_branch]
 225                 update_stream["csit-list"] = GenericScalar(
 226                     "!include:", "csit-jobs-{}.lst".format(release_on_current_branch)
 227                 )
 228
 229             # Update csit-mri-list parameters for next release
 230             if streams[0][release_on_current_branch].get("csit-mri-list"):
 231                 update_stream = streams[0][release_on_current_branch]
 232                 update_stream["csit-mri-list"] = "{{csit-mri-list-{}}}".format(
 233                     release_on_current_branch
 234                 )
 235
 236             # Update csit-weekly-list parameters for next release
 237             if streams[0][release_on_current_branch].get("csit-weekly-list"):
 238                 update_stream = streams[0][release_on_current_branch]
 239                 update_stream["csit-weekly-list"] = "{{csit-weekly-list-{}}}".format(
 240                     release_on_current_branch
 241                 )
 242
 243             # Update csit-sanity-list parameters for next release
 244             if streams[0][release_on_current_branch].get("csit-sanity-list"):
 245                 update_stream = streams[0][release_on_current_branch]
 246                 update_stream["csit-sanity-list"] = "{{csit-sanity-list-{}}}".format(
 247                     release_on_current_branch
 248                 )
 249
 250     with open(integration_test_jobs_yaml, "w") as f:
 251         stream = ruamel.yaml.round_trip_dump(data)
 252         f.write("---\n")
 253         f.write(stream)
 254
 255     # Update the csit-*-list variables in defaults.yaml
 256
 257     defaults_yaml = os.path.join(job_dir, "defaults.yaml")
 258
 259     with open(defaults_yaml, "r") as f:
 260         data = yaml.load(f)
 261
 262         # Add next release csit-mri-list-RELEASE
 263         new_csit_mri_list = copy.deepcopy(
 264             data[0]["defaults"].get("csit-mri-list-{}".format(release_on_stable_branch))
 265         )
 266         data[0]["defaults"][
 267             "csit-mri-list-{}".format(release_on_current_branch)
 268         ] = new_csit_mri_list.replace(
 269             release_on_stable_branch, release_on_current_branch
 270         )
 271
 272         # Add next release csit-mri-list-RELEASE
 273         new_csit_mri_list = copy.deepcopy(
 274             data[0]["defaults"].get("csit-mri-list-{}".format(release_on_stable_branch))
 275         )
 276         data[0]["defaults"][
 277             "csit-mri-list-{}".format(release_on_current_branch)
 278         ] = new_csit_mri_list.replace(
 279             release_on_stable_branch, release_on_current_branch
 280         )
 281
 282         # Add next release csit-weekly-list-RELEASE
 283         new_csit_mri_list = copy.deepcopy(
 284             data[0]["defaults"].get(
 285                 "csit-weekly-list-{}".format(release_on_stable_branch)
 286             )
 287         )
 288         data[0]["defaults"][
 289             "csit-weekly-list-{}".format(release_on_current_branch)
 290         ] = new_csit_mri_list.replace(
 291             release_on_stable_branch, release_on_current_branch
 292         )
 293
 294         # Add next release csit-sanity-list-RELEASE
 295         new_csit_mri_list = copy.deepcopy(
 296             data[0]["defaults"].get(
 297                 "csit-sanity-list-{}".format(release_on_stable_branch)
 298             )
 299         )
 300         data[0]["defaults"][
 301             "csit-sanity-list-{}".format(release_on_current_branch)
 302         ] = new_csit_mri_list.replace(
 303             release_on_stable_branch, release_on_current_branch
 304         )
 305
 306     with open(defaults_yaml, "w") as f:
 307         stream = ruamel.yaml.round_trip_dump(data)
 308         f.write("---\n")
 309         f.write(stream)
 310
 311     # Handle copying and updating the csit-*.lst files
 312     csit_file = "csit-jobs-{}.lst".format(release_on_stable_branch)
 313     src = os.path.join(job_dir, "integration", csit_file)
 314     dest = os.path.join(
 315         job_dir,
 316         "integration",
 317         csit_file.replace(release_on_stable_branch, release_on_current_branch),
 318     )
 319     shutil.copyfile(src, dest)
 320     with fileinput.FileInput(dest, inplace=True) as file:
 321         for line in file:
 322             print(
 323                 line.replace(release_on_stable_branch, release_on_current_branch),
 324                 end="",
 325             )
 326
 327
 328 parser = argparse.ArgumentParser(
 329     description="""Creates & updates jobs for ODL projects when branch cutting.
 330
 331     Example usage: python scripts/cut-branch.sh Silicon Phosphorus jjb/
 332
 333     ** If calling from tox the JOD_DIR is auto-detected so only pass the current
 334     and next release stream name. **
 335     """,
 336     formatter_class=RawTextHelpFormatter,
 337 )
 338 parser.add_argument(
 339     "release_on_stable_branch",
 340     metavar="RELEASE_ON_STABLE_BRANCH",
 341     type=str,
 342     help="The ODL release codename for the stable branch that was cut.",
 343 )
 344 parser.add_argument(
 345     "release_on_current_branch",
 346     metavar="RELEASE_ON_CURRENT_BRANCH",
 347     type=str,
 348     help="""The ODL release codename for the new {}
 349         (eg. Aluminium, Silicon).""".format(
 350         default_branch
 351     ),
 352 )
 353 parser.add_argument(
 354     "job_dir",
 355     metavar="JOB_DIR",
 356     type=str,
 357     help="Path to the directory containing JJB config.",
 358 )
 359 args = parser.parse_args()
 360
 361 # We only handle lower release codenames
 362 release_on_stable_branch = args.release_on_stable_branch.lower()
 363 release_on_current_branch = args.release_on_current_branch.lower()
 364
 365 create_and_update_project_jobs(
 366     release_on_stable_branch, release_on_current_branch, args.job_dir
 367 )
 368 update_job_streams(release_on_stable_branch, release_on_current_branch, args.job_dir)
 369 update_integration_csit_list(
 370     release_on_stable_branch, release_on_current_branch, args.job_dir
 371 )