14 1. What about the time between when a merge is submitted and
15 the patch is in the distribution? Should we look at the other
16 events and see when the merge job finished?
17 2. Use the git query option to request records in multiple queries
18 rather than grabbing all 50 in one shot. Keep going until the requested
19 number is found. Verify if this is better than just doing all 50 in one
20 shot since multiple requests are ssh round trips per request.
23 # This file started as an exact copy of git-review so including it"s copyright
26 Copyright (C) 2011-2017 OpenStack LLC.
28 Licensed under the Apache License, Version 2.0 (the "License");
29 you may not use this file except in compliance with the License.
30 You may obtain a copy of the License at
32 http://www.apache.org/licenses/LICENSE-2.0
34 Unless required by applicable law or agreed to in writing, software
35 distributed under the License is distributed on an "AS IS" BASIS,
36 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
39 See the License for the specific language governing permissions and
40 limitations under the License."""
43 logger = logging.getLogger("changes")
44 logger.setLevel(logging.DEBUG)
45 formatter = logging.Formatter(
46 "%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s"
48 ch = logging.StreamHandler()
49 ch.setLevel(logging.INFO)
50 ch.setFormatter(formatter)
52 fh = logging.FileHandler("/tmp/changes.txt", "w")
53 fh.setLevel(logging.DEBUG)
54 fh.setFormatter(formatter)
58 class ChangeId(object):
59 def __init__(self, changeid, merged):
60 self.changeid = changeid
64 class Changes(object):
79 VERBOSE = logging.INFO
80 DISTRO_PATH = "/tmp/distribution-karaf"
82 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
88 distro_path = DISTRO_PATH
89 distro_url = DISTRO_URL
90 project_names = PROJECT_NAMES
94 remote_url = REMOTE_URL
104 distro_path=DISTRO_PATH,
107 project_names=PROJECT_NAMES,
108 remote_url=REMOTE_URL,
112 self.distro_path = distro_path
115 self.project_names = project_names
116 self.remote_url = remote_url
117 self.verbose = verbose
119 self.set_log_level(verbose)
120 self.regex_changeid = re.compile(
121 r"(Change-Id.*: (\bI[a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)"
123 # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))')
124 self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")')
125 self.regex_shortmsg2 = re.compile(r"(git.commit.message.short=(.*))")
126 self.regex_longmsg = re.compile(r"git.commit.message.full=(.*)")
127 self.regex_commitid = re.compile(r"(git.commit.id=(.*))")
130 def set_log_level(level):
133 def epoch_to_utc(self, epoch):
134 utc = time.gmtime(epoch)
136 return time.strftime("%Y-%m-%d %H:%M:%S", utc)
138 def pretty_print_gerrits(self, project, gerrits):
140 print("%s" % project)
141 print("i grantedOn lastUpdatd chang subject")
143 "-- ------------------- ------------------- ----- -----------------------------------------"
146 print("gerrit is under review")
148 for i, gerrit in enumerate(gerrits):
149 if isinstance(gerrit, dict):
151 "%02d %19s %19s %5s %s"
154 self.epoch_to_utc(gerrit["grantedOn"])
155 if "grantedOn" in gerrit
157 self.epoch_to_utc(gerrit["lastUpdated"])
158 if "lastUpdated" in gerrit
160 gerrit["number"] if "number" in gerrit else "00000",
161 gerrit["subject"].encode("ascii", "replace")
162 if "subject" in gerrit
167 def pretty_print_projects(self, projects):
168 print("========================================")
170 print("========================================")
171 if isinstance(projects, dict):
172 for project_name, values in sorted(projects.items()):
173 if "includes" in values:
174 self.pretty_print_gerrits(project_name, values["includes"])
176 def set_projects(self, project_names=PROJECT_NAMES):
177 for project in project_names:
178 self.projects[project] = {"commit": [], "includes": []}
180 def download_distro(self):
182 Download the distribution from self.distro_url and extract it to self.distro_path
185 "attempting to download distribution from %s and extract to %s",
190 tmp_distro_zip = "/tmp/distro.zip"
191 tmp_unzipped_location = "/tmp/distro_unzipped"
192 downloader = urllib3.PoolManager(cert_reqs="CERT_NONE")
194 # disabling warnings to prevent scaring the user with InsecureRequestWarning
195 urllib3.disable_warnings()
197 downloaded_distro = downloader.request("GET", self.distro_url)
198 with open(tmp_distro_zip, "wb") as f:
199 f.write(downloaded_distro.data)
201 downloaded_distro.release_conn()
203 # after the .zip is extracted we want to rename it to be the distro_path which may have
204 # been given by the user
205 distro_zip = zipfile.ZipFile(tmp_distro_zip, "r")
206 distro_zip.extractall(tmp_unzipped_location)
207 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
209 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
210 # there is relevant (and maybe already put there by this tool earlier)
213 tmp_unzipped_location + "/" + unzipped_distro_folder[0],
219 "Unable to move extracted files from %s to %s. Using whatever bits are already there",
220 tmp_unzipped_location,
224 def get_includes(self, project, changeid=None, msg=None, merged=True):
226 Get the gerrits that would be included before the change merge time.
228 :param str project: The project to search
229 :param str or None changeid: The Change-Id of the gerrit to use for the merge time
230 :param str or None msg: The commit message of the gerrit to use for the merge time
231 :param bool merged: The requested gerrit was merged
232 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
235 includes = self.gerritquery.get_gerrits(
236 project, changeid, 1, msg, status="merged"
239 includes = self.gerritquery.get_gerrits(
240 project, changeid, 1, None, None, True
244 "Review %s in %s:%s was not found",
247 self.gerritquery.branch,
251 gerrits = self.gerritquery.get_gerrits(
252 project, changeid=None, limit=self.qlimit, msg=msg, status="merged"
254 for gerrit in gerrits:
255 # don"t include the same change in the list
256 if gerrit["id"] == changeid:
259 # TODO: should the check be < or <=?
260 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
261 includes.append(gerrit)
263 # break out if we have the number requested
264 if len(includes) == self.limit + 1:
267 if len(includes) != self.limit + 1:
269 "%s query limit was not large enough to capture %d gerrits",
277 def extract_gitproperties_file(fullpath):
279 Extract a git.properties from a jar archive.
281 :param str fullpath: Path to the jar
282 :return str: Containing git.properties or None if not found
284 if zipfile.is_zipfile(fullpath):
285 zf = zipfile.ZipFile(fullpath, "r")
287 pfile = zf.open("META-INF/git.properties")
288 return str(pfile.read())
293 def get_changeid_from_properties(self, project, pfile):
295 Parse the git.properties file to find a Change-Id.
297 There are a few different forms that we know of so far:
298 - I0123456789012345678901234567890123456789
300 - no Change-Id at all. There is a commit message and commit hash.
301 In this example the commit hash cannot be found because it was a merge
302 so you must use the message. Note spaces need to be replaced with 's.
303 - a patch that has not been merged. For these we look at the gerrit comment
304 for when the patch-test job starts.
306 :param str project: The project to search
307 :param str pfile: String containing the content of the git.properties file
308 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
310 logger.info("trying Change-Id from git.properties in %s", project)
311 # match a 40 or 8 char Change-Id hash. both start with I
312 changeid = self.regex_changeid.search(pfile)
313 if changeid and changeid.group(2):
315 "trying Change-Id from git.properties as merged in %s: %s",
320 gerrits = self.gerritquery.get_gerrits(
321 project, changeid.group(2), 1, None, status="merged"
325 "found Change-Id from git.properties as merged in %s", project
327 return ChangeId(changeid.group(2), True)
329 # Maybe this is a patch that has not merged yet
331 "did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s",
336 gerrits = self.gerritquery.get_gerrits(
337 project, changeid.group(2), 1, None, status=None, comments=True
341 "found Change-Id from git.properties as unmerged in %s", project
343 return ChangeId(gerrits[0]["id"], False)
346 "did not find Change-Id from git.properties in %s, trying commitid", project
349 # match a git commit id
350 commitid = self.regex_commitid.search(pfile)
351 if commitid and commitid.group(2):
353 "trying commitid from git.properties in %s: %s",
358 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
361 "found Change-Id from git.properties as unmerged in %s", project
363 return ChangeId(gerrits[0]["id"], True)
366 "did not find Change-Id from commitid from git.properties in %s, trying short commit message1",
370 # Didn't find a Change-Id so try to get a commit message
371 # match on "blah" but only keep the blah
372 msg = self.regex_shortmsg1.search(pfile)
373 if msg and msg.group(2):
374 # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2))
376 "trying with short commit-msg 1 from git.properties in %s: %s",
381 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
384 "found Change-Id from git.properties short commit-msg 1 in %s",
387 return ChangeId(gerrits[0]["id"], True)
389 msg_no_spaces = msg.group(2).replace(" ", "+")
391 "did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s",
396 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
399 "found Change-Id from git.properties short commit-msg 1 (no spaces) in %s",
402 return ChangeId(gerrits[0]["id"], True)
405 "did not find Change-Id from short commit message1 from git.properties in %s",
409 # Didn't find a Change-Id so try to get a commit message
410 # match on "blah" but only keep the blah
411 msg = self.regex_shortmsg2.search(pfile)
412 if msg and msg.group(2):
414 "trying with short commit-msg 2 from git.properties in %s: %s",
419 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
422 "found Change-Id from git.properties short commit-msg 2 in %s",
425 return ChangeId(gerrits[0]["id"], True)
427 msg_no_spaces = msg.group(2).replace(" ", "+")
429 "did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s",
434 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
437 "found Change-Id from git.properties short commit-msg 2 (no spaces) in %s",
440 return ChangeId(gerrits[0]["id"], True)
443 "did not find Change-Id from short commit message2 from git.properties in %s",
447 # Maybe one of the monster 'merge the world' gerrits
448 msg = self.regex_longmsg.search(pfile)
451 lines = str(msg.group()).split("\\n")
453 (i for i, line in enumerate(lines[:-1]) if "* changes\\:" in line), None
455 first_msg = lines[cli + 1] if cli else None
458 "did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s",
462 gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
465 "found Change-Id from git.properties merge commit-msg in %s",
468 return ChangeId(gerrits[0]["id"], True)
470 logger.warn("did not find Change-Id for %s" % project)
472 return ChangeId(None, False)
474 def find_distro_changeid(self, project):
476 Find a distribution Change-Id by finding a project jar in
477 the distribution and parsing it's git.properties.
479 :param str project: The project to search
480 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
482 project_dir = os.path.join(
483 self.distro_path, "system", "org", "opendaylight", project
486 for root, dirs, files in os.walk(project_dir):
488 if file_.endswith(".jar"):
489 fullpath = os.path.join(root, file_)
490 pfile = self.extract_gitproperties_file(fullpath)
492 changeid = self.get_changeid_from_properties(project, pfile)
493 if changeid.changeid:
497 "Could not find %s Change-Id in git.properties", project
499 break # all jars will have the same git.properties
500 if pfile is not None:
501 break # all jars will have the same git.properties
503 logger.warn("Could not find a git.properties file for %s", project)
504 return ChangeId(None, False)
506 def get_taglist(self):
508 Read a taglist.log file into memory
510 :return taglist: The taglist.log file read into memory
512 tagfile = os.path.join(self.distro_path, "taglist.log")
514 # Ensure the file exists and then read it
515 if os.path.isfile(tagfile):
516 with open(tagfile, "r") as fp:
520 def find_project_commit_changeid(self, taglist, project):
522 Find a commit id for the given project
524 :param str taglist: the taglist.log file read into memory
525 :param str project: The project to search
526 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
528 # break the regex up since {} is a valid regex element but we need it for the format project
529 re1 = r"({0} ".format(project)
530 re1 = re1 + r"(\b[a-f0-9]{40})\b|\b([a-f0-9]{8})\b" + r")"
531 commitid = re.search(re1, taglist)
532 if commitid and commitid.group(2):
534 "trying commitid from taglist.log in %s: %s", project, commitid.group(2)
537 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
539 logger.info("found Change-Id from taglist.log as merged in %s", project)
540 return ChangeId(gerrits[0]["id"], True)
543 "did not find Change-Id from commitid from taglist.log in %s", project
545 return ChangeId(None, False)
548 self.gerritquery = gerritquery.GerritQuery(
549 self.remote_url, self.branch, self.qlimit, self.verbose
551 self.set_projects(self.project_names)
553 def print_options(self):
555 "Using these options: branch: %s, limit: %d, qlimit: %d"
556 % (self.branch, self.limit, self.qlimit)
558 print("remote_url: %s" % self.remote_url)
559 print("distro_path: %s" % self.distro_path)
560 print("projects: %s" % (", ".join(map(str, self.projects))))
562 "gerrit 00 is the most recent patch from which the project was built followed by the next most"
563 " recently merged patches up to %s." % self.limit
568 Internal wrapper between main, options parser and internal code.
570 Get the gerrit for the given Change-Id and parse it.
571 Loop over all projects:
572 get qlimit gerrits and parse them
573 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
575 # TODO: need method to validate the branch matches the distribution
580 if self.distro_url is not None:
581 self.download_distro()
584 "Checking if this is an autorelease build by looking for taglist.log"
586 taglist = self.get_taglist()
587 if taglist is not None:
588 for project in sorted(self.projects):
589 logger.info("Processing %s using taglist.log", project)
590 changeid = self.find_project_commit_changeid(taglist, project)
591 if changeid.changeid:
592 self.projects[project]["commit"] = changeid.changeid
593 self.projects[project]["includes"] = self.get_includes(
594 project, changeid.changeid, msg=None, merged=changeid.merged
599 "This is not an autorelease build, continuing as integration distribution"
601 for project in sorted(self.projects):
602 logger.info("Processing %s", project)
603 changeid = self.find_distro_changeid(project)
604 if changeid.changeid:
605 self.projects[project]["commit"] = changeid.changeid
606 self.projects[project]["includes"] = self.get_includes(
607 project, changeid.changeid, msg=None, merged=changeid.merged
612 parser = argparse.ArgumentParser(description=COPYRIGHT)
618 help="git branch for patch under test",
624 default=self.DISTRO_PATH,
625 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH,
631 default=self.DISTRO_URL,
632 help="optional url to download a distribution " + str(self.DISTRO_URL),
640 help="number of gerrits to return",
646 default=self.PROJECT_NAMES,
647 help="list of projects to include in output",
654 default=self.QUERY_LIMIT,
655 help="number of gerrits to search",
661 default=self.REMOTE_URL,
662 help="git remote url to use for gerrit",
669 default=self.VERBOSE,
670 help="Output more information about what's going on",
676 help="Print the license and exit",
682 version="%s version %s" % (os.path.split(sys.argv[0])[-1], 0.1),
685 options = parser.parse_args()
691 self.branch = options.branch
692 self.distro_path = options.distro_path
693 self.distro_url = options.distro_url
694 self.limit = options.limit
695 self.qlimit = options.qlimit
696 self.remote_url = options.remote_url
697 self.verbose = options.verbose
698 if options.projects != self.PROJECT_NAMES:
699 self.project_names = options.projects.split(",")
701 # TODO: add check to verify that the remote can be reached,
702 # though the first gerrit query will fail anyways
704 projects = self.run_cmd()
705 self.pretty_print_projects(projects)
713 except Exception as e:
714 # If one does unguarded print(e) here, in certain locales the implicit
715 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
716 # range(128)". See rhbz#1058167.
720 # Python 3, we"re home free.
723 logger.warn(u.encode("utf-8"))
725 sys.exit(getattr(e, "EXIT_CODE", -1))
728 if __name__ == "__main__":