14 1. What about the time between when a merge is submitted and
15 the patch is in the distribution? Should we look at the other
16 events and see when the merge job finished?
17 2. Use the git query option to request records in multiple queries
18 rather than grabbing all 50 in one shot. Keep going until the requested
19 number is found. Verify if this is better than just doing all 50 in one
20 shot since multiple requests are ssh round trips per request.
23 # This file started as an exact copy of git-review so including it"s copyright
26 Copyright (C) 2011-2017 OpenStack LLC.
28 Licensed under the Apache License, Version 2.0 (the "License");
29 you may not use this file except in compliance with the License.
30 You may obtain a copy of the License at
32 http://www.apache.org/licenses/LICENSE-2.0
34 Unless required by applicable law or agreed to in writing, software
35 distributed under the License is distributed on an "AS IS" BASIS,
36 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
39 See the License for the specific language governing permissions and
40 limitations under the License."""
43 logger = logging.getLogger("changes")
44 logger.setLevel(logging.DEBUG)
45 formatter = logging.Formatter('%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s')
46 ch = logging.StreamHandler()
47 ch.setLevel(logging.INFO)
48 ch.setFormatter(formatter)
50 fh = logging.FileHandler("/tmp/changes.txt", "w")
51 fh.setLevel(logging.DEBUG)
52 fh.setFormatter(formatter)
56 class ChangeId(object):
57 def __init__(self, changeid, merged):
58 self.changeid = changeid
62 class Changes(object):
63 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
64 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
65 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
66 NETVIRT_PROJECTS = ["netvirt", "controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf",
67 "neutron", "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"]
68 PROJECT_NAMES = NETVIRT_PROJECTS
69 VERBOSE = logging.INFO
70 DISTRO_PATH = "/tmp/distribution-karaf"
72 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
78 distro_path = DISTRO_PATH
79 distro_url = DISTRO_URL
80 project_names = PROJECT_NAMES
84 remote_url = REMOTE_URL
91 def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH,
92 limit=LIMIT, qlimit=QUERY_LIMIT,
93 project_names=PROJECT_NAMES, remote_url=REMOTE_URL,
96 self.distro_path = distro_path
99 self.project_names = project_names
100 self.remote_url = remote_url
101 self.verbose = verbose
103 self.set_log_level(verbose)
104 self.regex_changeid = re.compile(r'(Change-Id.*: (\bI[a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)')
105 # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))')
106 self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")')
107 self.regex_shortmsg2 = re.compile(r'(git.commit.message.short=(.*))')
108 self.regex_longmsg = re.compile(r'git.commit.message.full=(.*)')
109 self.regex_commitid = re.compile(r'(git.commit.id=(.*))')
112 def set_log_level(level):
115 def epoch_to_utc(self, epoch):
116 utc = time.gmtime(epoch)
118 return time.strftime("%Y-%m-%d %H:%M:%S", utc)
120 def pretty_print_gerrits(self, project, gerrits):
122 print("%s" % project)
123 print("i grantedOn lastUpdatd chang subject")
124 print("-- ------------------- ------------------- ----- -----------------------------------------")
126 print("gerrit is under review")
128 for i, gerrit in enumerate(gerrits):
129 if isinstance(gerrit, dict):
130 print("%02d %19s %19s %5s %s"
132 self.epoch_to_utc(gerrit["grantedOn"]) if "grantedOn" in gerrit else 0,
133 self.epoch_to_utc(gerrit["lastUpdated"]) if "lastUpdated" in gerrit else 0,
134 gerrit["number"] if "number" in gerrit else "00000",
135 gerrit["subject"].encode('ascii', 'replace') if "subject" in gerrit else "none"))
137 def pretty_print_projects(self, projects):
138 print("========================================")
140 print("========================================")
141 if isinstance(projects, dict):
142 for project_name, values in sorted(projects.items()):
143 if "includes" in values:
144 self.pretty_print_gerrits(project_name, values["includes"])
146 def set_projects(self, project_names=PROJECT_NAMES):
147 for project in project_names:
148 self.projects[project] = {"commit": [], "includes": []}
150 def download_distro(self):
152 Download the distribution from self.distro_url and extract it to self.distro_path
154 logger.info("attempting to download distribution from %s and extract to %s", self.distro_url, self.distro_path)
156 tmp_distro_zip = '/tmp/distro.zip'
157 tmp_unzipped_location = '/tmp/distro_unzipped'
158 downloader = urllib3.PoolManager(cert_reqs='CERT_NONE')
160 # disabling warnings to prevent scaring the user with InsecureRequestWarning
161 urllib3.disable_warnings()
163 downloaded_distro = downloader.request('GET', self.distro_url)
164 with open(tmp_distro_zip, 'wb') as f:
165 f.write(downloaded_distro.data)
167 downloaded_distro.release_conn()
169 # after the .zip is extracted we want to rename it to be the distro_path which may have
170 # been given by the user
171 distro_zip = zipfile.ZipFile(tmp_distro_zip, 'r')
172 distro_zip.extractall(tmp_unzipped_location)
173 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
175 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
176 # there is relevant (and maybe already put there by this tool earlier)
178 os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path)
181 logger.warn("Unable to move extracted files from %s to %s. Using whatever bits are already there",
182 tmp_unzipped_location, self.distro_path)
184 def get_includes(self, project, changeid=None, msg=None, merged=True):
186 Get the gerrits that would be included before the change merge time.
188 :param str project: The project to search
189 :param str or None changeid: The Change-Id of the gerrit to use for the merge time
190 :param str or None msg: The commit message of the gerrit to use for the merge time
191 :param bool merged: The requested gerrit was merged
192 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
195 includes = self.gerritquery.get_gerrits(project, changeid, 1, msg, status="merged")
197 includes = self.gerritquery.get_gerrits(project, changeid, 1, None, None, True)
199 logger.info("Review %s in %s:%s was not found", changeid, project, self.gerritquery.branch)
202 gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg, status="merged")
203 for gerrit in gerrits:
204 # don"t include the same change in the list
205 if gerrit["id"] == changeid:
208 # TODO: should the check be < or <=?
209 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
210 includes.append(gerrit)
212 # break out if we have the number requested
213 if len(includes) == self.limit + 1:
216 if len(includes) != self.limit + 1:
217 logger.info("%s query limit was not large enough to capture %d gerrits", project, self.limit)
222 def extract_gitproperties_file(fullpath):
224 Extract a git.properties from a jar archive.
226 :param str fullpath: Path to the jar
227 :return str: Containing git.properties or None if not found
229 if zipfile.is_zipfile(fullpath):
230 zf = zipfile.ZipFile(fullpath, "r")
232 pfile = zf.open("META-INF/git.properties")
233 return str(pfile.read())
238 def get_changeid_from_properties(self, project, pfile):
240 Parse the git.properties file to find a Change-Id.
242 There are a few different forms that we know of so far:
243 - I0123456789012345678901234567890123456789
245 - no Change-Id at all. There is a commit message and commit hash.
246 In this example the commit hash cannot be found because it was a merge
247 so you must use the message. Note spaces need to be replaced with 's.
248 - a patch that has not been merged. For these we look at the gerrit comment
249 for when the patch-test job starts.
251 :param str project: The project to search
252 :param str pfile: String containing the content of the git.properties file
253 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
255 logger.info("trying Change-Id from git.properties in %s", project)
256 # match a 40 or 8 char Change-Id hash. both start with I
257 changeid = self.regex_changeid.search(pfile)
258 if changeid and changeid.group(2):
259 logger.info("trying Change-Id from git.properties as merged in %s: %s", project, changeid.group(2))
261 gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status="merged")
263 logger.info("found Change-Id from git.properties as merged in %s", project)
264 return ChangeId(changeid.group(2), True)
266 # Maybe this is a patch that has not merged yet
267 logger.info("did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s",
268 project, changeid.group(2))
270 gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status=None, comments=True)
272 logger.info("found Change-Id from git.properties as unmerged in %s", project)
273 return ChangeId(gerrits[0]["id"], False)
275 logger.info("did not find Change-Id from git.properties in %s, trying commitid", project)
277 # match a git commit id
278 commitid = self.regex_commitid.search(pfile)
279 if commitid and commitid.group(2):
280 logger.info("trying commitid from git.properties in %s: %s", project, commitid.group(2))
282 gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
284 logger.info("found Change-Id from git.properties as unmerged in %s", project)
285 return ChangeId(gerrits[0]["id"], True)
287 logger.info("did not find Change-Id from commitid from git.properties in %s, trying short commit message1",
290 # Didn't find a Change-Id so try to get a commit message
291 # match on "blah" but only keep the blah
292 msg = self.regex_shortmsg1.search(pfile)
293 if msg and msg.group(2):
294 # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2))
295 logger.info("trying with short commit-msg 1 from git.properties in %s: %s", project, msg.group(2))
297 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
299 logger.info("found Change-Id from git.properties short commit-msg 1 in %s", project)
300 return ChangeId(gerrits[0]["id"], True)
302 msg_no_spaces = msg.group(2).replace(" ", "+")
303 logger.info("did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s",
304 project, msg_no_spaces)
306 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
308 logger.info("found Change-Id from git.properties short commit-msg 1 (no spaces) in %s", project)
309 return ChangeId(gerrits[0]["id"], True)
311 logger.info("did not find Change-Id from short commit message1 from git.properties in %s", project)
313 # Didn't find a Change-Id so try to get a commit message
314 # match on "blah" but only keep the blah
315 msg = self.regex_shortmsg2.search(pfile)
316 if msg and msg.group(2):
317 logger.info("trying with short commit-msg 2 from git.properties in %s: %s", project, msg.group(2))
319 gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
321 logger.info("found Change-Id from git.properties short commit-msg 2 in %s", project)
322 return ChangeId(gerrits[0]["id"], True)
324 msg_no_spaces = msg.group(2).replace(" ", "+")
325 logger.info("did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s",
326 project, msg_no_spaces)
328 gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
330 logger.info("found Change-Id from git.properties short commit-msg 2 (no spaces) in %s", project)
331 return ChangeId(gerrits[0]["id"], True)
333 logger.info("did not find Change-Id from short commit message2 from git.properties in %s", project)
335 # Maybe one of the monster 'merge the world' gerrits
336 msg = self.regex_longmsg.search(pfile)
339 lines = str(msg.group()).split("\\n")
340 cli = next((i for i, line in enumerate(lines[:-1]) if '* changes\\:' in line), None)
341 first_msg = lines[cli + 1] if cli else None
343 logger.info("did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s",
345 gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
347 logger.info("found Change-Id from git.properties merge commit-msg in %s", project)
348 return ChangeId(gerrits[0]["id"], True)
350 logger.warn("did not find Change-Id for %s" % project)
352 return ChangeId(None, False)
354 def find_distro_changeid(self, project):
356 Find a distribution Change-Id by finding a project jar in
357 the distribution and parsing it's git.properties.
359 :param str project: The project to search
360 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
362 project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project)
364 for root, dirs, files in os.walk(project_dir):
366 if file_.endswith(".jar"):
367 fullpath = os.path.join(root, file_)
368 pfile = self.extract_gitproperties_file(fullpath)
370 changeid = self.get_changeid_from_properties(project, pfile)
371 if changeid.changeid:
374 logger.warn("Could not find %s Change-Id in git.properties", project)
375 break # all jars will have the same git.properties
376 if pfile is not None:
377 break # all jars will have the same git.properties
379 logger.warn("Could not find a git.properties file for %s", project)
380 return ChangeId(None, False)
383 self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose)
384 self.set_projects(self.project_names)
386 def print_options(self):
387 print("Using these options: branch: %s, limit: %d, qlimit: %d"
388 % (self.branch, self.limit, self.qlimit))
389 print("remote_url: %s" % self.remote_url)
390 print("distro_path: %s" % self.distro_path)
391 print("projects: %s" % (", ".join(map(str, self.projects))))
392 print("gerrit 00 is the most recent patch from which the project was built followed by the next most"
393 " recently merged patches up to %s." % self.limit)
397 Internal wrapper between main, options parser and internal code.
399 Get the gerrit for the given Change-Id and parse it.
400 Loop over all projects:
401 get qlimit gerrits and parse them
402 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
404 # TODO: need method to validate the branch matches the distribution
409 if self.distro_url is not None:
410 self.download_distro()
412 for project in sorted(self.projects):
413 logger.info("Processing %s", project)
414 changeid = self.find_distro_changeid(project)
415 if changeid.changeid:
416 self.projects[project]['commit'] = changeid.changeid
417 self.projects[project]["includes"] =\
418 self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged)
422 parser = argparse.ArgumentParser(description=COPYRIGHT)
424 parser.add_argument("-b", "--branch", default=self.BRANCH,
425 help="git branch for patch under test")
426 parser.add_argument("-d", "--distro-path", dest="distro_path", default=self.DISTRO_PATH,
427 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH)
428 parser.add_argument("-u", "--distro-url", dest="distro_url", default=self.DISTRO_URL,
429 help="optional url to download a distribution " + str(self.DISTRO_URL))
430 parser.add_argument("-l", "--limit", dest="limit", type=int, default=self.LIMIT,
431 help="number of gerrits to return")
432 parser.add_argument("-p", "--projects", dest="projects", default=self.PROJECT_NAMES,
433 help="list of projects to include in output")
434 parser.add_argument("-q", "--query-limit", dest="qlimit", type=int, default=self.QUERY_LIMIT,
435 help="number of gerrits to search")
436 parser.add_argument("-r", "--remote", dest="remote_url", default=self.REMOTE_URL,
437 help="git remote url to use for gerrit")
438 parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=self.VERBOSE,
439 help="Output more information about what's going on")
440 parser.add_argument("--license", dest="license", action="store_true",
441 help="Print the license and exit")
442 parser.add_argument("-V", "--version", action="version",
443 version="%s version %s" %
444 (os.path.split(sys.argv[0])[-1], 0.1))
446 options = parser.parse_args()
452 self.branch = options.branch
453 self.distro_path = options.distro_path
454 self.distro_url = options.distro_url
455 self.limit = options.limit
456 self.qlimit = options.qlimit
457 self.remote_url = options.remote_url
458 self.verbose = options.verbose
459 if options.projects != self.PROJECT_NAMES:
460 self.project_names = options.projects.split(',')
462 # TODO: add check to verify that the remote can be reached,
463 # though the first gerrit query will fail anyways
465 projects = self.run_cmd()
466 self.pretty_print_projects(projects)
474 except Exception as e:
475 # If one does unguarded print(e) here, in certain locales the implicit
476 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
477 # range(128)". See rhbz#1058167.
481 # Python 3, we"re home free.
484 logger.warn(u.encode("utf-8"))
486 sys.exit(getattr(e, "EXIT_CODE", -1))
489 if __name__ == "__main__":