14 1. What about the time between when a merge is submitted and
15 the patch is in the distribution? Should we look at the other
16 events and see when the merge job finished?
17 2. Use the git query option to request records in multiple queries
18 rather than grabbing all 50 in one shot. Keep going until the requested
19 number is found. Verify if this is better than just doing all 50 in one
20 shot since multiple requests are ssh round trips per request.
23 # This file started as an exact copy of git-review so including it"s copyright
26 Copyright (C) 2011-2017 OpenStack LLC.
28 Licensed under the Apache License, Version 2.0 (the "License");
29 you may not use this file except in compliance with the License.
30 You may obtain a copy of the License at
32 http://www.apache.org/licenses/LICENSE-2.0
34 Unless required by applicable law or agreed to in writing, software
35 distributed under the License is distributed on an "AS IS" BASIS,
36 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
39 See the License for the specific language governing permissions and
40 limitations under the License."""
43 logger = logging.getLogger("changes")
44 logger.setLevel(logging.DEBUG)
45 formatter = logging.Formatter('%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s')
46 ch = logging.StreamHandler()
47 ch.setLevel(logging.INFO)
48 ch.setFormatter(formatter)
50 fh = logging.FileHandler("/tmp/changes.txt", "w")
51 fh.setLevel(logging.DEBUG)
52 fh.setFormatter(formatter)
56 class ChangeId(object):
57 def __init__(self, changeid, merged):
58 self.changeid = changeid
62 class Changes(object):
63 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
64 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
65 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
66 NETVIRT_PROJECTS = ["netvirt", "controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf",
67 "neutron", "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"]
68 PROJECT_NAMES = NETVIRT_PROJECTS
69 VERBOSE = logging.INFO
70 DISTRO_PATH = "/tmp/distribution-karaf"
72 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
78 distro_path = DISTRO_PATH
79 distro_url = DISTRO_URL
80 project_names = PROJECT_NAMES
84 remote_url = REMOTE_URL
91 def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH,
92 limit=LIMIT, qlimit=QUERY_LIMIT,
93 project_names=PROJECT_NAMES, remote_url=REMOTE_URL,
96 self.distro_path = distro_path
99 self.project_names = project_names
100 self.remote_url = remote_url
101 self.verbose = verbose
103 self.set_log_level(verbose)
104 self.regex_changeid = re.compile(r'(Change-Id.*: (\bI[a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)')
105 # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))')
106 self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")')
107 self.regex_shortmsg2 = re.compile(r'(git.commit.message.short=(.*))')
108 self.regex_longmsg = re.compile(r'git.commit.message.full=(.*)')
109 self.regex_commitid = re.compile(r'(git.commit.id=(.*))')
112 def set_log_level(level):
115 def epoch_to_utc(self, epoch):
116 utc = time.gmtime(epoch)
118 return time.strftime("%Y-%m-%d %H:%M:%S", utc)
120 def pretty_print_gerrits(self, project, gerrits):
122 print("%s" % project)
123 print("i grantedOn lastUpdatd chang subject")
124 print("-- ------------------- ------------------- ----- -----------------------------------------")
126 print("gerrit is under review")
128 for i, gerrit in enumerate(gerrits):
129 if isinstance(gerrit, dict):
130 print("%02d %19s %19s %5s %s"
132 self.epoch_to_utc(gerrit["grantedOn"]) if "grantedOn" in gerrit else 0,
133 self.epoch_to_utc(gerrit["lastUpdated"]) if "lastUpdated" in gerrit else 0,
134 gerrit["number"] if "number" in gerrit else "00000",
135 gerrit["subject"].encode('ascii', 'replace') if "subject" in gerrit else "none"))
137 def pretty_print_projects(self, projects):
138 print("========================================")
140 print("========================================")
141 if isinstance(projects, dict):
142 for project_name, values in sorted(projects.items()):
143 if "includes" in values:
144 self.pretty_print_gerrits(project_name, values["includes"])
146 def set_projects(self, project_names=PROJECT_NAMES):
147 for project in project_names:
148 self.projects[project] = {"commit": [], "includes": []}
150 def download_distro(self):
152 Download the distribution from self.distro_url and extract it to self.distro_path
154 logger.info("attempting to download distribution from %s and extract to %s", self.distro_url, self.distro_path)
156 tmp_distro_zip = '/tmp/distro.zip'
157 tmp_unzipped_location = '/tmp/distro_unzipped'
158 downloader = urllib3.PoolManager(cert_reqs='CERT_NONE')
160 # disabling warnings to prevent scaring the user with InsecureRequestWarning
161 urllib3.disable_warnings()
163 downloaded_distro = downloader.request('GET', self.distro_url)
164 with open(tmp_distro_zip, 'wb') as f:
165 f.write(downloaded_distro.data)
167 downloaded_distro.release_conn()
169 # after the .zip is extracted we want to rename it to be the distro_path which may have
170 # been given by the user
171 distro_zip = zipfile.ZipFile(tmp_distro_zip, 'r')
172 distro_zip.extractall(tmp_unzipped_location)
173 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
175 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
176 # there is relevant (and maybe already put there by this tool earlier)
178 os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path)
181 logger.warn("Unable to move extracted files from %s to %s. Using whatever bits are already there",
182 tmp_unzipped_location, self.distro_path)
184 def get_includes(self, project, changeid=None, msg=None, merged=True):
186 Get the gerrits that would be included before the change merge time.
188 :param str project: The project to search
189 :param str or None changeid: The Change-Id of the gerrit to use for the merge time
190 :param str or None msg: The commit message of the gerrit to use for the merge time
191 :param bool merged: The requested gerrit was merged
192 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
195 includes = self.gerritquery.get_gerrits(project, changeid, 1, msg, status="merged")
197 includes = self.gerritquery.get_gerrits(project, changeid, 1, None, None, True)
199 logger.info("Review %s in %s:%s was not found", changeid, project, self.gerritquery.branch)
202 gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg, status="merged")
203 for gerrit in gerrits:
204 # don"t include the same change in the list
205 if gerrit["id"] == changeid:
208 # TODO: should the check be < or <=?
209 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
210 includes.append(gerrit)
212 # break out if we have the number requested
213 if len(includes) == self.limit + 1:
216 if len(includes) != self.limit + 1:
217 logger.info("%s query limit was not large enough to capture %d gerrits", project, self.limit)
222 def extract_gitproperties_file(fullpath):
224 Extract a git.properties from a jar archive.
226 :param str fullpath: Path to the jar
227 :return str: Containing git.properties or None if not found
229 if zipfile.is_zipfile(fullpath):
230 zf = zipfile.ZipFile(fullpath, "r")
232 pfile = zf.open("META-INF/git.properties")
233 return str(pfile.read())
238 def get_changeid_from_properties(self, project, pfile):
240 Parse the git.properties file to find a Change-Id.
242 There are a few different forms that we know of so far:
243 - I0123456789012345678901234567890123456789
245 - no Change-Id at all. There is a commit message and commit hash.
246 In this example the commit hash cannot be found because it was a merge
247 so you must use the message. Note spaces need to be replaced with 's.
248 - a patch that has not been merged. For these we look at the gerrit comment
249 for when the patch-test job starts.
251 :param str project: The project to search
252 :param str pfile: String containing the content of the git.properties file
253 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
255 logger.info("trying Change-Id from git.properties in %s", project)
256 # match a 40 or 8 char Change-Id hash. both start with I
257 changeid = self.regex_changeid.search(pfile)
258 if changeid and changeid.group(2):
259 logger.info("trying Change-Id from git.properties as merged in %s: %s", project, changeid.group(2))
261 gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status="merged")
263 logger.info("found Change-Id from git.properties as merged in %s", project)
264 return ChangeId(changeid.group(2), True)
266 # Maybe this is a patch that has not merged yet
267 logger.info("did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s",
268 project, changeid.group(2))
270 gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status=None, comments=True)
272 logger.info("found Change-Id from git.properties as unmerged in %s", project)
273 return ChangeId(gerrits[0]["id"], False)
275 logger.info("did not find Change-Id from git.properties in %s, trying commitid", project)
277 # match a git commit id
278 commitid = self.regex_commitid.search(pfile)
279 if commitid and commitid.group(2):
280 logger.info("trying commitid from git.properties in %s: %s", project, commitid.group(2))
282 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
284 logger.info("found Change-Id from git.properties as unmerged in %s", project)
285 return ChangeId(gerrits[0]["id"], True)
287 logger.info("did not find Change-Id from commitid from git.properties in %s, trying short commit message1",
290 # Didn't find a Change-Id so try to get a commit message
291 # match on "blah" but only keep the blah
292 msg = self.regex_shortmsg1.search(pfile)
293 if msg and msg.group(2):
294 # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2))
295 logger.info("trying with short commit-msg 1 from git.properties in %s: %s", project, msg.group(2))
297 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
299 logger.info("found Change-Id from git.properties short commit-msg 1 in %s", project)
300 return ChangeId(gerrits[0]["id"], True)
302 msg_no_spaces = msg.group(2).replace(" ", "+")
303 logger.info("did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s",
304 project, msg_no_spaces)
306 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
308 logger.info("found Change-Id from git.properties short commit-msg 1 (no spaces) in %s", project)
309 return ChangeId(gerrits[0]["id"], True)
311 logger.info("did not find Change-Id from short commit message1 from git.properties in %s", project)
313 # Didn't find a Change-Id so try to get a commit message
314 # match on "blah" but only keep the blah
315 msg = self.regex_shortmsg2.search(pfile)
316 if msg and msg.group(2):
317 logger.info("trying with short commit-msg 2 from git.properties in %s: %s", project, msg.group(2))
319 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
321 logger.info("found Change-Id from git.properties short commit-msg 2 in %s", project)
322 return ChangeId(gerrits[0]["id"], True)
324 msg_no_spaces = msg.group(2).replace(" ", "+")
325 logger.info("did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s",
326 project, msg_no_spaces)
328 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
330 logger.info("found Change-Id from git.properties short commit-msg 2 (no spaces) in %s", project)
331 return ChangeId(gerrits[0]["id"], True)
333 logger.info("did not find Change-Id from short commit message2 from git.properties in %s", project)
335 # Maybe one of the monster 'merge the world' gerrits
336 msg = self.regex_longmsg.search(pfile)
339 lines = str(msg.group()).split("\\n")
340 cli = next((i for i, line in enumerate(lines[:-1]) if '* changes\\:' in line), None)
341 first_msg = lines[cli + 1] if cli else None
343 logger.info("did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s",
345 gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
347 logger.info("found Change-Id from git.properties merge commit-msg in %s", project)
348 return ChangeId(gerrits[0]["id"], True)
350 logger.warn("did not find Change-Id for %s" % project)
352 return ChangeId(None, False)
354 def find_distro_changeid(self, project):
356 Find a distribution Change-Id by finding a project jar in
357 the distribution and parsing it's git.properties.
359 :param str project: The project to search
360 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
362 project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project)
364 for root, dirs, files in os.walk(project_dir):
366 if file_.endswith(".jar"):
367 fullpath = os.path.join(root, file_)
368 pfile = self.extract_gitproperties_file(fullpath)
370 changeid = self.get_changeid_from_properties(project, pfile)
371 if changeid.changeid:
374 logger.warn("Could not find %s Change-Id in git.properties", project)
375 break # all jars will have the same git.properties
376 if pfile is not None:
377 break # all jars will have the same git.properties
379 logger.warn("Could not find a git.properties file for %s", project)
380 return ChangeId(None, False)
382 def get_taglist(self):
384 Read a taglist.log file into memory
386 :return taglist: The taglist.log file read into memory
388 tagfile = os.path.join(self.distro_path, "taglist.log")
390 # Ensure the file exists and then read it
391 if os.path.isfile(tagfile):
392 with open(tagfile, 'r') as fp:
396 def find_project_commit_changeid(self, taglist, project):
398 Find a commit id for the given project
400 :param str taglist: the taglist.log file read into memory
401 :param str project: The project to search
402 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
404 # break the regex up since {} is a valid regex element but we need it for the format project
405 re1 = r'({0} '.format(project)
406 re1 = re1 + r'(\b[a-f0-9]{40})\b|\b([a-f0-9]{8})\b' + r')'
407 commitid = re.search(re1, taglist)
408 if commitid and commitid.group(2):
409 logger.info("trying commitid from taglist.log in %s: %s", project, commitid.group(2))
411 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
413 logger.info("found Change-Id from taglist.log as merged in %s", project)
414 return ChangeId(gerrits[0]["id"], True)
416 logger.warn("did not find Change-Id from commitid from taglist.log in %s", project)
417 return ChangeId(None, False)
420 self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose)
421 self.set_projects(self.project_names)
423 def print_options(self):
424 print("Using these options: branch: %s, limit: %d, qlimit: %d"
425 % (self.branch, self.limit, self.qlimit))
426 print("remote_url: %s" % self.remote_url)
427 print("distro_path: %s" % self.distro_path)
428 print("projects: %s" % (", ".join(map(str, self.projects))))
429 print("gerrit 00 is the most recent patch from which the project was built followed by the next most"
430 " recently merged patches up to %s." % self.limit)
434 Internal wrapper between main, options parser and internal code.
436 Get the gerrit for the given Change-Id and parse it.
437 Loop over all projects:
438 get qlimit gerrits and parse them
439 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
441 # TODO: need method to validate the branch matches the distribution
446 if self.distro_url is not None:
447 self.download_distro()
449 logger.info("Checking if this is an autorelease build by looking for taglist.log")
450 taglist = self.get_taglist()
451 if taglist is not None:
452 for project in sorted(self.projects):
453 logger.info("Processing %s using taglist.log", project)
454 changeid = self.find_project_commit_changeid(taglist, project)
455 if changeid.changeid:
456 self.projects[project]['commit'] = changeid.changeid
457 self.projects[project]["includes"] = \
458 self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged)
461 logger.info("This is not an autorelease build, continuing as integration distribution")
462 for project in sorted(self.projects):
463 logger.info("Processing %s", project)
464 changeid = self.find_distro_changeid(project)
465 if changeid.changeid:
466 self.projects[project]['commit'] = changeid.changeid
467 self.projects[project]["includes"] =\
468 self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged)
472 parser = argparse.ArgumentParser(description=COPYRIGHT)
474 parser.add_argument("-b", "--branch", default=self.BRANCH,
475 help="git branch for patch under test")
476 parser.add_argument("-d", "--distro-path", dest="distro_path", default=self.DISTRO_PATH,
477 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH)
478 parser.add_argument("-u", "--distro-url", dest="distro_url", default=self.DISTRO_URL,
479 help="optional url to download a distribution " + str(self.DISTRO_URL))
480 parser.add_argument("-l", "--limit", dest="limit", type=int, default=self.LIMIT,
481 help="number of gerrits to return")
482 parser.add_argument("-p", "--projects", dest="projects", default=self.PROJECT_NAMES,
483 help="list of projects to include in output")
484 parser.add_argument("-q", "--query-limit", dest="qlimit", type=int, default=self.QUERY_LIMIT,
485 help="number of gerrits to search")
486 parser.add_argument("-r", "--remote", dest="remote_url", default=self.REMOTE_URL,
487 help="git remote url to use for gerrit")
488 parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=self.VERBOSE,
489 help="Output more information about what's going on")
490 parser.add_argument("--license", dest="license", action="store_true",
491 help="Print the license and exit")
492 parser.add_argument("-V", "--version", action="version",
493 version="%s version %s" %
494 (os.path.split(sys.argv[0])[-1], 0.1))
496 options = parser.parse_args()
502 self.branch = options.branch
503 self.distro_path = options.distro_path
504 self.distro_url = options.distro_url
505 self.limit = options.limit
506 self.qlimit = options.qlimit
507 self.remote_url = options.remote_url
508 self.verbose = options.verbose
509 if options.projects != self.PROJECT_NAMES:
510 self.project_names = options.projects.split(',')
512 # TODO: add check to verify that the remote can be reached,
513 # though the first gerrit query will fail anyways
515 projects = self.run_cmd()
516 self.pretty_print_projects(projects)
524 except Exception as e:
525 # If one does unguarded print(e) here, in certain locales the implicit
526 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
527 # range(128)". See rhbz#1058167.
531 # Python 3, we"re home free.
534 logger.warn(u.encode("utf-8"))
536 sys.exit(getattr(e, "EXIT_CODE", -1))
539 if __name__ == "__main__":