1 # SPDX-License-Identifier: EPL-1.0
2 ##############################################################################
3 # Copyright (c) 2020 Thanh Ha
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Eclipse Public License v1.0
7 # which accompanies this distribution, and is available at
8 # http://www.eclipse.org/legal/epl-v10.html
10 ##############################################################################
11 """Script for cutting new jobs when branching a new stable release."""
14 from argparse import RawTextHelpFormatter
23 except ModuleNotFoundError:
24 print("ERROR: This script requires the package 'ruamel.yaml', please install it.")
26 "If ruamel.yaml is not available in your system's package manager you"
27 " can install from PyPi with:"
30 print(" pip install --user ruamel.yaml")
33 yaml = ruamel.yaml.YAML()
34 yaml.allow_duplicate_keys = True
35 yaml.preserve_quotes = True
37 default_branch = "master" # This is the primary dev branch of the project
40 def create_and_update_project_jobs(
41 release_on_stable_branch, release_on_current_branch, job_dir
43 """Create and update project build jobs for the current and next dev release.
45 Project jobs are jobs defined in the project.yaml that have the same name
46 the directory they are in.
48 Only updates projects where the top project configuration has a name that
49 is equivalent to the current release. For example project name
50 "aaa-sulfur" would have a release that matches what was passed to
51 release_on_stable_branch.
53 for directory in filter(
54 lambda x: os.path.isdir(os.path.join(job_dir, x)), os.listdir(job_dir)
58 os.path.join(job_dir, directory, "{}.yaml".format(directory)), "r"
62 # Only create new jobs if the top level project name matches
63 # release_on_stable_branch variable
64 if not data[0]["project"]["name"] == "{}-{}".format(
65 directory, release_on_stable_branch
69 # Create a new job for the next release on the default_branch
70 new_job = copy.deepcopy(data[0])
71 new_job["project"]["name"] = "{}-{}".format(
72 directory, release_on_current_branch
74 new_job["project"]["branch"] = default_branch
75 new_job["project"]["stream"] = "{}".format(release_on_current_branch)
77 # Update exiting job for the new stable branch
78 data[0]["project"]["branch"] = "stable/{}".format(
79 release_on_stable_branch
82 data.insert(0, new_job)
85 os.path.join(job_dir, directory, "{}.yaml".format(directory)), "w"
87 stream = ruamel.yaml.round_trip_dump(data)
90 except FileNotFoundError: # If project.yaml file does not exist we can skip
94 def update_job_streams(release_on_stable_branch, release_on_current_branch, job_dir):
95 """Update projects that have a stream variable that is a list.
97 If a stream variable is a list that means the project likely has multiple
98 maintainance branches supported.
100 This function also does not support {project}.yaml files as parsing those
101 are handled by other functions in this script.
103 Only updates projects where the top stream in the list is equivalent to the
104 current release. For example stream "sulfur" would have a release that
105 matches what was passed to release_on_stable_branch.
107 for directory in filter(
108 lambda d: os.path.isdir(os.path.join(job_dir, d)), os.listdir(job_dir)
110 for job_file in filter(
111 lambda f: os.path.isfile(os.path.join(job_dir, directory, f)),
112 os.listdir(os.path.join(job_dir, directory)),
115 # Projects may have non-yaml files in their repos so ignore them.
116 if not job_file.endswith(".yaml"):
119 # Ignore project.yaml files as they are not supported by this function.
120 if job_file == "{}.yaml".format(directory):
125 with open(os.path.join(job_dir, directory, job_file), "r") as f:
129 streams = project.get("project", {}).get("stream", None)
131 if not isinstance(streams, list): # We only support lists streams
134 # Skip if the stream does not match
135 # release_on_stable_branch in the first item
136 if not streams[0].get(release_on_stable_branch, None):
139 # Create the next release stream
141 new_stream[release_on_current_branch] = copy.deepcopy(
142 streams[0].get(release_on_stable_branch)
145 # Update the previous release stream branch to
146 # stable/{stream} instead of default_branch
147 streams[0][release_on_stable_branch]["branch"] = "stable/{}".format(
148 release_on_stable_branch
151 streams.insert(0, new_stream)
154 # Because we are looping every file we only want to save if we made changes.
156 with open(os.path.join(job_dir, directory, job_file), "w") as f:
157 stream = ruamel.yaml.round_trip_dump(data)
162 def update_integration_csit_list(
163 release_on_stable_branch, release_on_current_branch, job_dir
165 """Update csit-*-list variables and files integration-test-jobs.yaml."""
168 def __init__(self, tag, value, style=None):
173 class GenericScalar(Generic):
175 def to_yaml(self, representer, node):
176 return representer.represent_scalar(node._tag, node._value)
179 def construct(constructor, node):
180 return constructor.construct_scalar(node)
182 def default_constructor(constructor, tag_suffix, node):
183 generic = {ruamel.yaml.ScalarNode: GenericScalar,}.get( # noqa
187 raise NotImplementedError("Node: " + str(type(node)))
188 style = getattr(node, "style", None)
189 instance = generic.__new__(generic)
191 state = generic.construct(constructor, node)
192 instance.__init__(tag_suffix, state, style=style)
194 ruamel.yaml.add_multi_constructor(
195 "", default_constructor, Loader=ruamel.yaml.SafeLoader
197 yaml.register_class(GenericScalar)
199 integration_test_jobs_yaml = os.path.join(
200 job_dir, "integration", "integration-test-jobs.yaml"
203 with open(integration_test_jobs_yaml, "r") as f:
207 # Skip items that are not of "project" type
208 if not project.get("project"):
211 streams = project.get("project", {}).get("stream", None)
213 # Skip projects that do not have a stream configured
214 if not isinstance(streams, list): # We only support lists streams
217 # Skip if the stream does not match
218 # release_on_current_branch in the first item
219 if not streams[0].get(release_on_current_branch, None):
222 # Update csit-list parameters for next release
223 if streams[0][release_on_current_branch].get("csit-list"):
224 update_stream = streams[0][release_on_current_branch]
225 update_stream["csit-list"] = GenericScalar(
226 "!include:", "csit-jobs-{}.lst".format(release_on_current_branch)
229 # Update csit-mri-list parameters for next release
230 if streams[0][release_on_current_branch].get("csit-mri-list"):
231 update_stream = streams[0][release_on_current_branch]
232 update_stream["csit-mri-list"] = "{{csit-mri-list-{}}}".format(
233 release_on_current_branch
236 # Update csit-weekly-list parameters for next release
237 if streams[0][release_on_current_branch].get("csit-weekly-list"):
238 update_stream = streams[0][release_on_current_branch]
239 update_stream["csit-weekly-list"] = "{{csit-weekly-list-{}}}".format(
240 release_on_current_branch
243 # Update csit-sanity-list parameters for next release
244 if streams[0][release_on_current_branch].get("csit-sanity-list"):
245 update_stream = streams[0][release_on_current_branch]
246 update_stream["csit-sanity-list"] = "{{csit-sanity-list-{}}}".format(
247 release_on_current_branch
250 with open(integration_test_jobs_yaml, "w") as f:
251 stream = ruamel.yaml.round_trip_dump(data)
255 # Update the csit-*-list variables in defaults.yaml
257 defaults_yaml = os.path.join(job_dir, "defaults.yaml")
259 with open(defaults_yaml, "r") as f:
262 # Add next release csit-mri-list-RELEASE
263 new_csit_mri_list = copy.deepcopy(
264 data[0]["defaults"].get("csit-mri-list-{}".format(release_on_stable_branch))
267 "csit-mri-list-{}".format(release_on_current_branch)
268 ] = new_csit_mri_list.replace(
269 release_on_stable_branch, release_on_current_branch
272 # Add next release csit-mri-list-RELEASE
273 new_csit_mri_list = copy.deepcopy(
274 data[0]["defaults"].get("csit-mri-list-{}".format(release_on_stable_branch))
277 "csit-mri-list-{}".format(release_on_current_branch)
278 ] = new_csit_mri_list.replace(
279 release_on_stable_branch, release_on_current_branch
282 # Add next release csit-weekly-list-RELEASE
283 new_csit_mri_list = copy.deepcopy(
284 data[0]["defaults"].get(
285 "csit-weekly-list-{}".format(release_on_stable_branch)
289 "csit-weekly-list-{}".format(release_on_current_branch)
290 ] = new_csit_mri_list.replace(
291 release_on_stable_branch, release_on_current_branch
294 # Add next release csit-sanity-list-RELEASE
295 new_csit_mri_list = copy.deepcopy(
296 data[0]["defaults"].get(
297 "csit-sanity-list-{}".format(release_on_stable_branch)
301 "csit-sanity-list-{}".format(release_on_current_branch)
302 ] = new_csit_mri_list.replace(
303 release_on_stable_branch, release_on_current_branch
306 with open(defaults_yaml, "w") as f:
307 stream = ruamel.yaml.round_trip_dump(data)
311 # Handle copying and updating the csit-*.lst files
312 csit_file = "csit-jobs-{}.lst".format(release_on_stable_branch)
313 src = os.path.join(job_dir, "integration", csit_file)
317 csit_file.replace(release_on_stable_branch, release_on_current_branch),
319 shutil.copyfile(src, dest)
320 with fileinput.FileInput(dest, inplace=True) as file:
323 line.replace(release_on_stable_branch, release_on_current_branch),
328 parser = argparse.ArgumentParser(
329 description="""Creates & updates jobs for ODL projects when branch cutting.
331 Example usage: python scripts/cut-branch.sh Phosphorus Sulfur jjb/
333 ** If calling from tox the JOD_DIR is auto-detected so only pass the current
334 and next release stream name. **
336 formatter_class=RawTextHelpFormatter,
339 "release_on_stable_branch",
340 metavar="RELEASE_ON_STABLE_BRANCH",
342 help="The ODL release codename for the stable branch that was cut.",
345 "release_on_current_branch",
346 metavar="RELEASE_ON_CURRENT_BRANCH",
348 help="""The ODL release codename for the new {}
349 (eg. Sulfur, Phosphorus).""".format(
357 help="Path to the directory containing JJB config.",
359 args = parser.parse_args()
361 # We only handle lower release codenames
362 release_on_stable_branch = args.release_on_stable_branch.lower()
363 release_on_current_branch = args.release_on_current_branch.lower()
365 create_and_update_project_jobs(
366 release_on_stable_branch, release_on_current_branch, args.job_dir
368 update_job_streams(release_on_stable_branch, release_on_current_branch, args.job_dir)
369 update_integration_csit_list(
370 release_on_stable_branch, release_on_current_branch, args.job_dir