14 1. What about the time between when a merge is submitted and
15 the patch is in the distribution? Should we look at the other
16 events and see when the merge job finished?
17 2. Use the git query option to request records in multiple queries
18 rather than grabbing all 50 in one shot. Keep going until the requested
19 number is found. Verify if this is better than just doing all 50 in one
20 shot since multiple requests are ssh round trips per request.
23 # This file started as an exact copy of git-review so including it"s copyright
26 Copyright (C) 2011-2017 OpenStack LLC.
28 Licensed under the Apache License, Version 2.0 (the "License");
29 you may not use this file except in compliance with the License.
30 You may obtain a copy of the License at
32 http://www.apache.org/licenses/LICENSE-2.0
34 Unless required by applicable law or agreed to in writing, software
35 distributed under the License is distributed on an "AS IS" BASIS,
36 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
39 See the License for the specific language governing permissions and
40 limitations under the License."""
43 logger = logging.getLogger("changes")
44 logger.setLevel(logging.DEBUG)
45 formatter = logging.Formatter(
46 "%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s"
48 ch = logging.StreamHandler()
49 ch.setLevel(logging.INFO)
50 ch.setFormatter(formatter)
52 fh = logging.FileHandler("/tmp/changes.txt", "w")
53 fh.setLevel(logging.DEBUG)
54 fh.setFormatter(formatter)
58 class ChangeId(object):
59 def __init__(self, changeid, merged):
60 self.changeid = changeid
64 class Changes(object):
65 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
66 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
67 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
84 PROJECT_NAMES = NETVIRT_PROJECTS
85 VERBOSE = logging.INFO
86 DISTRO_PATH = "/tmp/distribution-karaf"
88 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
94 distro_path = DISTRO_PATH
95 distro_url = DISTRO_URL
96 project_names = PROJECT_NAMES
100 remote_url = REMOTE_URL
103 regex_changeid = None
104 regex_shortmsg = None
110 distro_path=DISTRO_PATH,
113 project_names=PROJECT_NAMES,
114 remote_url=REMOTE_URL,
118 self.distro_path = distro_path
121 self.project_names = project_names
122 self.remote_url = remote_url
123 self.verbose = verbose
125 self.set_log_level(verbose)
126 self.regex_changeid = re.compile(
127 r"(Change-Id.*: (\bI[a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)"
129 # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))')
130 self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")')
131 self.regex_shortmsg2 = re.compile(r"(git.commit.message.short=(.*))")
132 self.regex_longmsg = re.compile(r"git.commit.message.full=(.*)")
133 self.regex_commitid = re.compile(r"(git.commit.id=(.*))")
136 def set_log_level(level):
139 def epoch_to_utc(self, epoch):
140 utc = time.gmtime(epoch)
142 return time.strftime("%Y-%m-%d %H:%M:%S", utc)
144 def pretty_print_gerrits(self, project, gerrits):
146 print("%s" % project)
147 print("i grantedOn lastUpdatd chang subject")
149 "-- ------------------- ------------------- ----- -----------------------------------------"
152 print("gerrit is under review")
154 for i, gerrit in enumerate(gerrits):
155 if isinstance(gerrit, dict):
157 "%02d %19s %19s %5s %s"
160 self.epoch_to_utc(gerrit["grantedOn"])
161 if "grantedOn" in gerrit
163 self.epoch_to_utc(gerrit["lastUpdated"])
164 if "lastUpdated" in gerrit
166 gerrit["number"] if "number" in gerrit else "00000",
167 gerrit["subject"].encode("ascii", "replace")
168 if "subject" in gerrit
173 def pretty_print_projects(self, projects):
174 print("========================================")
176 print("========================================")
177 if isinstance(projects, dict):
178 for project_name, values in sorted(projects.items()):
179 if "includes" in values:
180 self.pretty_print_gerrits(project_name, values["includes"])
182 def set_projects(self, project_names=PROJECT_NAMES):
183 for project in project_names:
184 self.projects[project] = {"commit": [], "includes": []}
186 def download_distro(self):
188 Download the distribution from self.distro_url and extract it to self.distro_path
191 "attempting to download distribution from %s and extract to %s",
196 tmp_distro_zip = "/tmp/distro.zip"
197 tmp_unzipped_location = "/tmp/distro_unzipped"
198 downloader = urllib3.PoolManager(cert_reqs="CERT_NONE")
200 # disabling warnings to prevent scaring the user with InsecureRequestWarning
201 urllib3.disable_warnings()
203 downloaded_distro = downloader.request("GET", self.distro_url)
204 with open(tmp_distro_zip, "wb") as f:
205 f.write(downloaded_distro.data)
207 downloaded_distro.release_conn()
209 # after the .zip is extracted we want to rename it to be the distro_path which may have
210 # been given by the user
211 distro_zip = zipfile.ZipFile(tmp_distro_zip, "r")
212 distro_zip.extractall(tmp_unzipped_location)
213 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
215 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
216 # there is relevant (and maybe already put there by this tool earlier)
219 tmp_unzipped_location + "/" + unzipped_distro_folder[0],
225 "Unable to move extracted files from %s to %s. Using whatever bits are already there",
226 tmp_unzipped_location,
230 def get_includes(self, project, changeid=None, msg=None, merged=True):
232 Get the gerrits that would be included before the change merge time.
234 :param str project: The project to search
235 :param str or None changeid: The Change-Id of the gerrit to use for the merge time
236 :param str or None msg: The commit message of the gerrit to use for the merge time
237 :param bool merged: The requested gerrit was merged
238 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
241 includes = self.gerritquery.get_gerrits(
242 project, changeid, 1, msg, status="merged"
245 includes = self.gerritquery.get_gerrits(
246 project, changeid, 1, None, None, True
250 "Review %s in %s:%s was not found",
253 self.gerritquery.branch,
257 gerrits = self.gerritquery.get_gerrits(
258 project, changeid=None, limit=self.qlimit, msg=msg, status="merged"
260 for gerrit in gerrits:
261 # don"t include the same change in the list
262 if gerrit["id"] == changeid:
265 # TODO: should the check be < or <=?
266 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
267 includes.append(gerrit)
269 # break out if we have the number requested
270 if len(includes) == self.limit + 1:
273 if len(includes) != self.limit + 1:
275 "%s query limit was not large enough to capture %d gerrits",
283 def extract_gitproperties_file(fullpath):
285 Extract a git.properties from a jar archive.
287 :param str fullpath: Path to the jar
288 :return str: Containing git.properties or None if not found
290 if zipfile.is_zipfile(fullpath):
291 zf = zipfile.ZipFile(fullpath, "r")
293 pfile = zf.open("META-INF/git.properties")
294 return str(pfile.read())
299 def get_changeid_from_properties(self, project, pfile):
301 Parse the git.properties file to find a Change-Id.
303 There are a few different forms that we know of so far:
304 - I0123456789012345678901234567890123456789
306 - no Change-Id at all. There is a commit message and commit hash.
307 In this example the commit hash cannot be found because it was a merge
308 so you must use the message. Note spaces need to be replaced with 's.
309 - a patch that has not been merged. For these we look at the gerrit comment
310 for when the patch-test job starts.
312 :param str project: The project to search
313 :param str pfile: String containing the content of the git.properties file
314 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
316 logger.info("trying Change-Id from git.properties in %s", project)
317 # match a 40 or 8 char Change-Id hash. both start with I
318 changeid = self.regex_changeid.search(pfile)
319 if changeid and changeid.group(2):
321 "trying Change-Id from git.properties as merged in %s: %s",
326 gerrits = self.gerritquery.get_gerrits(
327 project, changeid.group(2), 1, None, status="merged"
331 "found Change-Id from git.properties as merged in %s", project
333 return ChangeId(changeid.group(2), True)
335 # Maybe this is a patch that has not merged yet
337 "did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s",
342 gerrits = self.gerritquery.get_gerrits(
343 project, changeid.group(2), 1, None, status=None, comments=True
347 "found Change-Id from git.properties as unmerged in %s", project
349 return ChangeId(gerrits[0]["id"], False)
352 "did not find Change-Id from git.properties in %s, trying commitid", project
355 # match a git commit id
356 commitid = self.regex_commitid.search(pfile)
357 if commitid and commitid.group(2):
359 "trying commitid from git.properties in %s: %s",
364 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
367 "found Change-Id from git.properties as unmerged in %s", project
369 return ChangeId(gerrits[0]["id"], True)
372 "did not find Change-Id from commitid from git.properties in %s, trying short commit message1",
376 # Didn't find a Change-Id so try to get a commit message
377 # match on "blah" but only keep the blah
378 msg = self.regex_shortmsg1.search(pfile)
379 if msg and msg.group(2):
380 # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2))
382 "trying with short commit-msg 1 from git.properties in %s: %s",
387 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
390 "found Change-Id from git.properties short commit-msg 1 in %s",
393 return ChangeId(gerrits[0]["id"], True)
395 msg_no_spaces = msg.group(2).replace(" ", "+")
397 "did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s",
402 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
405 "found Change-Id from git.properties short commit-msg 1 (no spaces) in %s",
408 return ChangeId(gerrits[0]["id"], True)
411 "did not find Change-Id from short commit message1 from git.properties in %s",
415 # Didn't find a Change-Id so try to get a commit message
416 # match on "blah" but only keep the blah
417 msg = self.regex_shortmsg2.search(pfile)
418 if msg and msg.group(2):
420 "trying with short commit-msg 2 from git.properties in %s: %s",
425 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
428 "found Change-Id from git.properties short commit-msg 2 in %s",
431 return ChangeId(gerrits[0]["id"], True)
433 msg_no_spaces = msg.group(2).replace(" ", "+")
435 "did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s",
440 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
443 "found Change-Id from git.properties short commit-msg 2 (no spaces) in %s",
446 return ChangeId(gerrits[0]["id"], True)
449 "did not find Change-Id from short commit message2 from git.properties in %s",
453 # Maybe one of the monster 'merge the world' gerrits
454 msg = self.regex_longmsg.search(pfile)
457 lines = str(msg.group()).split("\\n")
459 (i for i, line in enumerate(lines[:-1]) if "* changes\\:" in line), None
461 first_msg = lines[cli + 1] if cli else None
464 "did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s",
468 gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
471 "found Change-Id from git.properties merge commit-msg in %s",
474 return ChangeId(gerrits[0]["id"], True)
476 logger.warn("did not find Change-Id for %s" % project)
478 return ChangeId(None, False)
480 def find_distro_changeid(self, project):
482 Find a distribution Change-Id by finding a project jar in
483 the distribution and parsing it's git.properties.
485 :param str project: The project to search
486 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
488 project_dir = os.path.join(
489 self.distro_path, "system", "org", "opendaylight", project
492 for root, dirs, files in os.walk(project_dir):
494 if file_.endswith(".jar"):
495 fullpath = os.path.join(root, file_)
496 pfile = self.extract_gitproperties_file(fullpath)
498 changeid = self.get_changeid_from_properties(project, pfile)
499 if changeid.changeid:
503 "Could not find %s Change-Id in git.properties", project
505 break # all jars will have the same git.properties
506 if pfile is not None:
507 break # all jars will have the same git.properties
509 logger.warn("Could not find a git.properties file for %s", project)
510 return ChangeId(None, False)
512 def get_taglist(self):
514 Read a taglist.log file into memory
516 :return taglist: The taglist.log file read into memory
518 tagfile = os.path.join(self.distro_path, "taglist.log")
520 # Ensure the file exists and then read it
521 if os.path.isfile(tagfile):
522 with open(tagfile, "r") as fp:
526 def find_project_commit_changeid(self, taglist, project):
528 Find a commit id for the given project
530 :param str taglist: the taglist.log file read into memory
531 :param str project: The project to search
532 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
534 # break the regex up since {} is a valid regex element but we need it for the format project
535 re1 = r"({0} ".format(project)
536 re1 = re1 + r"(\b[a-f0-9]{40})\b|\b([a-f0-9]{8})\b" + r")"
537 commitid = re.search(re1, taglist)
538 if commitid and commitid.group(2):
540 "trying commitid from taglist.log in %s: %s", project, commitid.group(2)
543 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
545 logger.info("found Change-Id from taglist.log as merged in %s", project)
546 return ChangeId(gerrits[0]["id"], True)
549 "did not find Change-Id from commitid from taglist.log in %s", project
551 return ChangeId(None, False)
554 self.gerritquery = gerritquery.GerritQuery(
555 self.remote_url, self.branch, self.qlimit, self.verbose
557 self.set_projects(self.project_names)
559 def print_options(self):
561 "Using these options: branch: %s, limit: %d, qlimit: %d"
562 % (self.branch, self.limit, self.qlimit)
564 print("remote_url: %s" % self.remote_url)
565 print("distro_path: %s" % self.distro_path)
566 print("projects: %s" % (", ".join(map(str, self.projects))))
568 "gerrit 00 is the most recent patch from which the project was built followed by the next most"
569 " recently merged patches up to %s." % self.limit
574 Internal wrapper between main, options parser and internal code.
576 Get the gerrit for the given Change-Id and parse it.
577 Loop over all projects:
578 get qlimit gerrits and parse them
579 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
581 # TODO: need method to validate the branch matches the distribution
586 if self.distro_url is not None:
587 self.download_distro()
590 "Checking if this is an autorelease build by looking for taglist.log"
592 taglist = self.get_taglist()
593 if taglist is not None:
594 for project in sorted(self.projects):
595 logger.info("Processing %s using taglist.log", project)
596 changeid = self.find_project_commit_changeid(taglist, project)
597 if changeid.changeid:
598 self.projects[project]["commit"] = changeid.changeid
599 self.projects[project]["includes"] = self.get_includes(
600 project, changeid.changeid, msg=None, merged=changeid.merged
605 "This is not an autorelease build, continuing as integration distribution"
607 for project in sorted(self.projects):
608 logger.info("Processing %s", project)
609 changeid = self.find_distro_changeid(project)
610 if changeid.changeid:
611 self.projects[project]["commit"] = changeid.changeid
612 self.projects[project]["includes"] = self.get_includes(
613 project, changeid.changeid, msg=None, merged=changeid.merged
618 parser = argparse.ArgumentParser(description=COPYRIGHT)
624 help="git branch for patch under test",
630 default=self.DISTRO_PATH,
631 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH,
637 default=self.DISTRO_URL,
638 help="optional url to download a distribution " + str(self.DISTRO_URL),
646 help="number of gerrits to return",
652 default=self.PROJECT_NAMES,
653 help="list of projects to include in output",
660 default=self.QUERY_LIMIT,
661 help="number of gerrits to search",
667 default=self.REMOTE_URL,
668 help="git remote url to use for gerrit",
675 default=self.VERBOSE,
676 help="Output more information about what's going on",
682 help="Print the license and exit",
688 version="%s version %s" % (os.path.split(sys.argv[0])[-1], 0.1),
691 options = parser.parse_args()
697 self.branch = options.branch
698 self.distro_path = options.distro_path
699 self.distro_url = options.distro_url
700 self.limit = options.limit
701 self.qlimit = options.qlimit
702 self.remote_url = options.remote_url
703 self.verbose = options.verbose
704 if options.projects != self.PROJECT_NAMES:
705 self.project_names = options.projects.split(",")
707 # TODO: add check to verify that the remote can be reached,
708 # though the first gerrit query will fail anyways
710 projects = self.run_cmd()
711 self.pretty_print_projects(projects)
719 except Exception as e:
720 # If one does unguarded print(e) here, in certain locales the implicit
721 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
722 # range(128)". See rhbz#1058167.
726 # Python 3, we"re home free.
729 logger.warn(u.encode("utf-8"))
731 sys.exit(getattr(e, "EXIT_CODE", -1))
734 if __name__ == "__main__":