14 1. What about the time between when a merge is submitted and
15 the patch is in the distribution? Should we look at the other
16 events and see when the merge job finished?
17 2. Use the git query option to request records in multiple queries
18 rather than grabbing all 50 in one shot. Keep going until the requested
19 number is found. Verify if this is better than just doing all 50 in one
20 shot since multiple requests are ssh round trips per request.
23 # This file started as an exact copy of git-review so including it"s copyright
26 Copyright (C) 2011-2017 OpenStack LLC.
28 Licensed under the Apache License, Version 2.0 (the "License");
29 you may not use this file except in compliance with the License.
30 You may obtain a copy of the License at
32 http://www.apache.org/licenses/LICENSE-2.0
34 Unless required by applicable law or agreed to in writing, software
35 distributed under the License is distributed on an "AS IS" BASIS,
36 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
39 See the License for the specific language governing permissions and
40 limitations under the License."""
43 logger = logging.getLogger("changes")
44 logger.setLevel(logging.DEBUG)
45 formatter = logging.Formatter(
46 "%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s"
48 ch = logging.StreamHandler()
49 ch.setLevel(logging.INFO)
50 ch.setFormatter(formatter)
52 fh = logging.FileHandler("/tmp/changes.txt", "w")
53 fh.setLevel(logging.DEBUG)
54 fh.setFormatter(formatter)
58 class ChangeId(object):
59 def __init__(self, changeid, merged):
60 self.changeid = changeid
64 class Changes(object):
65 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
66 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
67 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
83 PROJECT_NAMES = NETVIRT_PROJECTS
84 VERBOSE = logging.INFO
85 DISTRO_PATH = "/tmp/distribution-karaf"
87 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
93 distro_path = DISTRO_PATH
94 distro_url = DISTRO_URL
95 project_names = PROJECT_NAMES
99 remote_url = REMOTE_URL
102 regex_changeid = None
103 regex_shortmsg = None
109 distro_path=DISTRO_PATH,
112 project_names=PROJECT_NAMES,
113 remote_url=REMOTE_URL,
117 self.distro_path = distro_path
120 self.project_names = project_names
121 self.remote_url = remote_url
122 self.verbose = verbose
124 self.set_log_level(verbose)
125 self.regex_changeid = re.compile(
126 r"(Change-Id.*: (\bI[a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)"
128 # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))')
129 self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")')
130 self.regex_shortmsg2 = re.compile(r"(git.commit.message.short=(.*))")
131 self.regex_longmsg = re.compile(r"git.commit.message.full=(.*)")
132 self.regex_commitid = re.compile(r"(git.commit.id=(.*))")
135 def set_log_level(level):
138 def epoch_to_utc(self, epoch):
139 utc = time.gmtime(epoch)
141 return time.strftime("%Y-%m-%d %H:%M:%S", utc)
143 def pretty_print_gerrits(self, project, gerrits):
145 print("%s" % project)
146 print("i grantedOn lastUpdatd chang subject")
148 "-- ------------------- ------------------- ----- -----------------------------------------"
151 print("gerrit is under review")
153 for i, gerrit in enumerate(gerrits):
154 if isinstance(gerrit, dict):
156 "%02d %19s %19s %5s %s"
159 self.epoch_to_utc(gerrit["grantedOn"])
160 if "grantedOn" in gerrit
162 self.epoch_to_utc(gerrit["lastUpdated"])
163 if "lastUpdated" in gerrit
165 gerrit["number"] if "number" in gerrit else "00000",
166 gerrit["subject"].encode("ascii", "replace")
167 if "subject" in gerrit
172 def pretty_print_projects(self, projects):
173 print("========================================")
175 print("========================================")
176 if isinstance(projects, dict):
177 for project_name, values in sorted(projects.items()):
178 if "includes" in values:
179 self.pretty_print_gerrits(project_name, values["includes"])
181 def set_projects(self, project_names=PROJECT_NAMES):
182 for project in project_names:
183 self.projects[project] = {"commit": [], "includes": []}
185 def download_distro(self):
187 Download the distribution from self.distro_url and extract it to self.distro_path
190 "attempting to download distribution from %s and extract to %s",
195 tmp_distro_zip = "/tmp/distro.zip"
196 tmp_unzipped_location = "/tmp/distro_unzipped"
197 downloader = urllib3.PoolManager(cert_reqs="CERT_NONE")
199 # disabling warnings to prevent scaring the user with InsecureRequestWarning
200 urllib3.disable_warnings()
202 downloaded_distro = downloader.request("GET", self.distro_url)
203 with open(tmp_distro_zip, "wb") as f:
204 f.write(downloaded_distro.data)
206 downloaded_distro.release_conn()
208 # after the .zip is extracted we want to rename it to be the distro_path which may have
209 # been given by the user
210 distro_zip = zipfile.ZipFile(tmp_distro_zip, "r")
211 distro_zip.extractall(tmp_unzipped_location)
212 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
214 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
215 # there is relevant (and maybe already put there by this tool earlier)
218 tmp_unzipped_location + "/" + unzipped_distro_folder[0],
224 "Unable to move extracted files from %s to %s. Using whatever bits are already there",
225 tmp_unzipped_location,
229 def get_includes(self, project, changeid=None, msg=None, merged=True):
231 Get the gerrits that would be included before the change merge time.
233 :param str project: The project to search
234 :param str or None changeid: The Change-Id of the gerrit to use for the merge time
235 :param str or None msg: The commit message of the gerrit to use for the merge time
236 :param bool merged: The requested gerrit was merged
237 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
240 includes = self.gerritquery.get_gerrits(
241 project, changeid, 1, msg, status="merged"
244 includes = self.gerritquery.get_gerrits(
245 project, changeid, 1, None, None, True
249 "Review %s in %s:%s was not found",
252 self.gerritquery.branch,
256 gerrits = self.gerritquery.get_gerrits(
257 project, changeid=None, limit=self.qlimit, msg=msg, status="merged"
259 for gerrit in gerrits:
260 # don"t include the same change in the list
261 if gerrit["id"] == changeid:
264 # TODO: should the check be < or <=?
265 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
266 includes.append(gerrit)
268 # break out if we have the number requested
269 if len(includes) == self.limit + 1:
272 if len(includes) != self.limit + 1:
274 "%s query limit was not large enough to capture %d gerrits",
282 def extract_gitproperties_file(fullpath):
284 Extract a git.properties from a jar archive.
286 :param str fullpath: Path to the jar
287 :return str: Containing git.properties or None if not found
289 if zipfile.is_zipfile(fullpath):
290 zf = zipfile.ZipFile(fullpath, "r")
292 pfile = zf.open("META-INF/git.properties")
293 return str(pfile.read())
298 def get_changeid_from_properties(self, project, pfile):
300 Parse the git.properties file to find a Change-Id.
302 There are a few different forms that we know of so far:
303 - I0123456789012345678901234567890123456789
305 - no Change-Id at all. There is a commit message and commit hash.
306 In this example the commit hash cannot be found because it was a merge
307 so you must use the message. Note spaces need to be replaced with 's.
308 - a patch that has not been merged. For these we look at the gerrit comment
309 for when the patch-test job starts.
311 :param str project: The project to search
312 :param str pfile: String containing the content of the git.properties file
313 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
315 logger.info("trying Change-Id from git.properties in %s", project)
316 # match a 40 or 8 char Change-Id hash. both start with I
317 changeid = self.regex_changeid.search(pfile)
318 if changeid and changeid.group(2):
320 "trying Change-Id from git.properties as merged in %s: %s",
325 gerrits = self.gerritquery.get_gerrits(
326 project, changeid.group(2), 1, None, status="merged"
330 "found Change-Id from git.properties as merged in %s", project
332 return ChangeId(changeid.group(2), True)
334 # Maybe this is a patch that has not merged yet
336 "did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s",
341 gerrits = self.gerritquery.get_gerrits(
342 project, changeid.group(2), 1, None, status=None, comments=True
346 "found Change-Id from git.properties as unmerged in %s", project
348 return ChangeId(gerrits[0]["id"], False)
351 "did not find Change-Id from git.properties in %s, trying commitid", project
354 # match a git commit id
355 commitid = self.regex_commitid.search(pfile)
356 if commitid and commitid.group(2):
358 "trying commitid from git.properties in %s: %s",
363 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
366 "found Change-Id from git.properties as unmerged in %s", project
368 return ChangeId(gerrits[0]["id"], True)
371 "did not find Change-Id from commitid from git.properties in %s, trying short commit message1",
375 # Didn't find a Change-Id so try to get a commit message
376 # match on "blah" but only keep the blah
377 msg = self.regex_shortmsg1.search(pfile)
378 if msg and msg.group(2):
379 # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2))
381 "trying with short commit-msg 1 from git.properties in %s: %s",
386 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
389 "found Change-Id from git.properties short commit-msg 1 in %s",
392 return ChangeId(gerrits[0]["id"], True)
394 msg_no_spaces = msg.group(2).replace(" ", "+")
396 "did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s",
401 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
404 "found Change-Id from git.properties short commit-msg 1 (no spaces) in %s",
407 return ChangeId(gerrits[0]["id"], True)
410 "did not find Change-Id from short commit message1 from git.properties in %s",
414 # Didn't find a Change-Id so try to get a commit message
415 # match on "blah" but only keep the blah
416 msg = self.regex_shortmsg2.search(pfile)
417 if msg and msg.group(2):
419 "trying with short commit-msg 2 from git.properties in %s: %s",
424 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
427 "found Change-Id from git.properties short commit-msg 2 in %s",
430 return ChangeId(gerrits[0]["id"], True)
432 msg_no_spaces = msg.group(2).replace(" ", "+")
434 "did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s",
439 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
442 "found Change-Id from git.properties short commit-msg 2 (no spaces) in %s",
445 return ChangeId(gerrits[0]["id"], True)
448 "did not find Change-Id from short commit message2 from git.properties in %s",
452 # Maybe one of the monster 'merge the world' gerrits
453 msg = self.regex_longmsg.search(pfile)
456 lines = str(msg.group()).split("\\n")
458 (i for i, line in enumerate(lines[:-1]) if "* changes\\:" in line), None
460 first_msg = lines[cli + 1] if cli else None
463 "did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s",
467 gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
470 "found Change-Id from git.properties merge commit-msg in %s",
473 return ChangeId(gerrits[0]["id"], True)
475 logger.warn("did not find Change-Id for %s" % project)
477 return ChangeId(None, False)
479 def find_distro_changeid(self, project):
481 Find a distribution Change-Id by finding a project jar in
482 the distribution and parsing it's git.properties.
484 :param str project: The project to search
485 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
487 project_dir = os.path.join(
488 self.distro_path, "system", "org", "opendaylight", project
491 for root, dirs, files in os.walk(project_dir):
493 if file_.endswith(".jar"):
494 fullpath = os.path.join(root, file_)
495 pfile = self.extract_gitproperties_file(fullpath)
497 changeid = self.get_changeid_from_properties(project, pfile)
498 if changeid.changeid:
502 "Could not find %s Change-Id in git.properties", project
504 break # all jars will have the same git.properties
505 if pfile is not None:
506 break # all jars will have the same git.properties
508 logger.warn("Could not find a git.properties file for %s", project)
509 return ChangeId(None, False)
511 def get_taglist(self):
513 Read a taglist.log file into memory
515 :return taglist: The taglist.log file read into memory
517 tagfile = os.path.join(self.distro_path, "taglist.log")
519 # Ensure the file exists and then read it
520 if os.path.isfile(tagfile):
521 with open(tagfile, "r") as fp:
525 def find_project_commit_changeid(self, taglist, project):
527 Find a commit id for the given project
529 :param str taglist: the taglist.log file read into memory
530 :param str project: The project to search
531 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
533 # break the regex up since {} is a valid regex element but we need it for the format project
534 re1 = r"({0} ".format(project)
535 re1 = re1 + r"(\b[a-f0-9]{40})\b|\b([a-f0-9]{8})\b" + r")"
536 commitid = re.search(re1, taglist)
537 if commitid and commitid.group(2):
539 "trying commitid from taglist.log in %s: %s", project, commitid.group(2)
542 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
544 logger.info("found Change-Id from taglist.log as merged in %s", project)
545 return ChangeId(gerrits[0]["id"], True)
548 "did not find Change-Id from commitid from taglist.log in %s", project
550 return ChangeId(None, False)
553 self.gerritquery = gerritquery.GerritQuery(
554 self.remote_url, self.branch, self.qlimit, self.verbose
556 self.set_projects(self.project_names)
558 def print_options(self):
560 "Using these options: branch: %s, limit: %d, qlimit: %d"
561 % (self.branch, self.limit, self.qlimit)
563 print("remote_url: %s" % self.remote_url)
564 print("distro_path: %s" % self.distro_path)
565 print("projects: %s" % (", ".join(map(str, self.projects))))
567 "gerrit 00 is the most recent patch from which the project was built followed by the next most"
568 " recently merged patches up to %s." % self.limit
573 Internal wrapper between main, options parser and internal code.
575 Get the gerrit for the given Change-Id and parse it.
576 Loop over all projects:
577 get qlimit gerrits and parse them
578 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
580 # TODO: need method to validate the branch matches the distribution
585 if self.distro_url is not None:
586 self.download_distro()
589 "Checking if this is an autorelease build by looking for taglist.log"
591 taglist = self.get_taglist()
592 if taglist is not None:
593 for project in sorted(self.projects):
594 logger.info("Processing %s using taglist.log", project)
595 changeid = self.find_project_commit_changeid(taglist, project)
596 if changeid.changeid:
597 self.projects[project]["commit"] = changeid.changeid
598 self.projects[project]["includes"] = self.get_includes(
599 project, changeid.changeid, msg=None, merged=changeid.merged
604 "This is not an autorelease build, continuing as integration distribution"
606 for project in sorted(self.projects):
607 logger.info("Processing %s", project)
608 changeid = self.find_distro_changeid(project)
609 if changeid.changeid:
610 self.projects[project]["commit"] = changeid.changeid
611 self.projects[project]["includes"] = self.get_includes(
612 project, changeid.changeid, msg=None, merged=changeid.merged
617 parser = argparse.ArgumentParser(description=COPYRIGHT)
623 help="git branch for patch under test",
629 default=self.DISTRO_PATH,
630 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH,
636 default=self.DISTRO_URL,
637 help="optional url to download a distribution " + str(self.DISTRO_URL),
645 help="number of gerrits to return",
651 default=self.PROJECT_NAMES,
652 help="list of projects to include in output",
659 default=self.QUERY_LIMIT,
660 help="number of gerrits to search",
666 default=self.REMOTE_URL,
667 help="git remote url to use for gerrit",
674 default=self.VERBOSE,
675 help="Output more information about what's going on",
681 help="Print the license and exit",
687 version="%s version %s" % (os.path.split(sys.argv[0])[-1], 0.1),
690 options = parser.parse_args()
696 self.branch = options.branch
697 self.distro_path = options.distro_path
698 self.distro_url = options.distro_url
699 self.limit = options.limit
700 self.qlimit = options.qlimit
701 self.remote_url = options.remote_url
702 self.verbose = options.verbose
703 if options.projects != self.PROJECT_NAMES:
704 self.project_names = options.projects.split(",")
706 # TODO: add check to verify that the remote can be reached,
707 # though the first gerrit query will fail anyways
709 projects = self.run_cmd()
710 self.pretty_print_projects(projects)
718 except Exception as e:
719 # If one does unguarded print(e) here, in certain locales the implicit
720 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
721 # range(128)". See rhbz#1058167.
725 # Python 3, we"re home free.
728 logger.warn(u.encode("utf-8"))
730 sys.exit(getattr(e, "EXIT_CODE", -1))
733 if __name__ == "__main__":