13 1. What about the time between when a merge is submitted and
14 the patch is in the distribution? Should we look at the other
15 events and see when the merge job finished?
16 2. Use the git query option to request records in multiple queries
17 rather than grabbing all 50 in one shot. Keep going until the requested
18 number is found. Verify if this is better than just doing all 50 in one
19 shot since multiple requests are ssh round trips per request.
22 # This file started as an exact copy of git-review so including it"s copyright
25 Copyright (C) 2011-2017 OpenStack LLC.
27 Licensed under the Apache License, Version 2.0 (the "License");
28 you may not use this file except in compliance with the License.
29 You may obtain a copy of the License at
31 http://www.apache.org/licenses/LICENSE-2.0
33 Unless required by applicable law or agreed to in writing, software
34 distributed under the License is distributed on an "AS IS" BASIS,
35 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
38 See the License for the specific language governing permissions and
39 limitations under the License."""
42 class ChangeId(object):
43 def __init__(self, changeid, merged):
44 self.changeid = changeid
48 class Changes(object):
49 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
50 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
51 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
52 NETVIRT_PROJECTS = ["netvirt", "controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf",
53 "neutron", "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"]
54 PROJECT_NAMES = NETVIRT_PROJECTS
56 DISTRO_PATH = "/tmp/distribution-karaf"
58 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
64 distro_path = DISTRO_PATH
65 distro_url = DISTRO_URL
66 project_names = PROJECT_NAMES
70 remote_url = REMOTE_URL
74 def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH,
75 limit=LIMIT, qlimit=QUERY_LIMIT,
76 project_names=PROJECT_NAMES, remote_url=REMOTE_URL,
79 self.distro_path = distro_path
82 self.project_names = project_names
83 self.remote_url = remote_url
84 self.verbose = verbose
87 def epoch_to_utc(self, epoch):
88 utc = time.gmtime(epoch)
90 return time.strftime("%Y-%m-%d %H:%M:%S", utc)
92 def pretty_print_gerrits(self, project, gerrits):
96 print("i grantedOn lastUpdatd chang subject")
97 print("-- ------------------- ------------------- ----- -----------------------------------------")
99 print("gerrit is under review")
101 for i, gerrit in enumerate(gerrits):
102 if isinstance(gerrit, dict):
103 print("%02d %19s %19s %5s %s"
105 self.epoch_to_utc(gerrit["grantedOn"]) if "grantedOn" in gerrit else 0,
106 self.epoch_to_utc(gerrit["lastUpdated"]) if "lastUpdated" in gerrit else 0,
107 gerrit["number"] if "number" in gerrit else "00000",
108 gerrit["subject"] if "subject" in gerrit else "none"))
110 def pretty_print_projects(self, projects):
111 if isinstance(projects, dict):
112 for project_name, values in sorted(projects.items()):
113 if "includes" in values:
114 self.pretty_print_gerrits(project_name, values["includes"])
116 def set_projects(self, project_names=PROJECT_NAMES):
117 for project in project_names:
118 self.projects[project] = {"commit": [], "includes": []}
120 def download_distro(self):
122 Download the distribution from self.distro_url and extract it to self.distro_path
124 if self.verbose >= 2:
125 print("attempting to download distribution from %s and extract to %s " %
126 (self.distro_url, self.distro_path))
128 tmp_distro_zip = '/tmp/distro.zip'
129 tmp_unzipped_location = '/tmp/distro_unzipped'
130 downloader = urllib3.PoolManager(cert_reqs='CERT_NONE')
132 # disabling warnings to prevent scaring the user with InsecureRequestWarning
133 urllib3.disable_warnings()
135 downloaded_distro = downloader.request('GET', self.distro_url)
136 with open(tmp_distro_zip, 'wb') as f:
137 f.write(downloaded_distro.data)
139 downloaded_distro.release_conn()
141 # after the .zip is extracted we want to rename it to be the distro_path which may have
142 # been given by the user
143 distro_zip = zipfile.ZipFile(tmp_distro_zip, 'r')
144 distro_zip.extractall(tmp_unzipped_location)
145 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
147 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
148 # there is relevant (and maybe already put there by this tool earlier)
150 os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path)
153 print("Unable to move extracted files from %s to %s. Using whatever bits are already there" %
154 (tmp_unzipped_location, self.distro_path))
156 def get_includes(self, project, changeid=None, msg=None, merged=True):
158 Get the gerrits that would be included before the change merge time.
160 :param str project: The project to search
161 :param str or None changeid: The Change-Id of the gerrit to use for the merge time
162 :param str or None msg: The commit message of the gerrit to use for the merge time
163 :param bool merged: The requested gerrit was merged
164 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
167 includes = self.gerritquery.get_gerrits(project, changeid, 1, msg, status="merged")
169 includes = self.gerritquery.get_gerrits(project, changeid, 1, None, None, True)
171 print("Review %s in %s:%s was not found" % (changeid, project, self.gerritquery.branch))
174 gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg, status="merged")
175 for gerrit in gerrits:
176 # don"t include the same change in the list
177 if gerrit["id"] == changeid:
180 # TODO: should the check be < or <=?
181 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
182 includes.append(gerrit)
184 # break out if we have the number requested
185 if len(includes) == self.limit + 1:
188 if len(includes) != self.limit + 1:
189 print("%s query limit was not large enough to capture %d gerrits" % (project, self.limit))
194 def extract_gitproperties_file(fullpath):
196 Extract a git.properties from a jar archive.
198 :param str fullpath: Path to the jar
199 :return str: Containing git.properties or None if not found
201 if zipfile.is_zipfile(fullpath):
202 zf = zipfile.ZipFile(fullpath, "r")
204 pfile = zf.open("META-INF/git.properties")
205 return str(pfile.read())
210 def get_changeid_from_properties(self, project, pfile):
212 Parse the git.properties file to find a Change-Id.
214 There are a few different forms that we know of so far:
215 - I0123456789012345678901234567890123456789
217 - no Change-Id at all. There is a commit message and commit hash.
218 In this example the commit hash cannot be found because it was a merge
219 so you must use the message. Note spaces need to be replaced with 's.
220 - a patch that has not been merged. For these we look at the gerrit comment
221 for when the patch-test job starts.
223 :param str project: The project to search
224 :param str pfile: String containing the content of the git.properties file
225 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
227 # match a 40 or 8 char Change-Id hash. both start with I
228 regex = re.compile(r'\bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b')
229 changeid = regex.search(pfile)
231 if self.verbose >= 1:
232 print("trying Change-Id as merged in %s" % (project))
234 gerrits = self.gerritquery.get_gerrits(project, changeid.group(), 1, None, status="merged")
236 return ChangeId(changeid.group(), True)
238 # Maybe this is a patch that has not merged yet
239 if self.verbose >= 1:
240 print("did not find Change-Id as merged in %s, trying as unmerged" % project)
242 gerrits = self.gerritquery.get_gerrits(project, changeid.group(), 1, None, status=None, comments=True)
244 return ChangeId(gerrits[0]["id"], False)
246 # Didn't find a Change-Id so try to get a commit message
247 # match on "blah" but only keep the blah
248 regex_msg = re.compile(r'"([^"]*)"|^git.commit.message.short=(.*)$')
249 msg = regex_msg.search(pfile)
251 if self.verbose >= 1:
252 print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group()))
254 gerrits = self.gerritquery.get_gerrits(project, None, 1, msg.group())
256 return ChangeId(gerrits[0]["id"], True)
258 msg_no_spaces = msg.group().replace(" ", "+")
259 if self.verbose >= 1:
260 print("did not find Change-Id in %s, trying with commit-msg (no spaces): %s" % (project, msg_no_spaces))
262 gerrits = self.gerritquery.get_gerrits(project, None, 1, msg_no_spaces)
264 return ChangeId(gerrits[0]["id"], True)
266 # Maybe one of the monster 'merge the world' gerrits
267 regex_msg = re.compile(r'git.commit.message.full=(.*)')
268 msg = regex_msg.search(pfile)
271 lines = str(msg.group()).split("\\n")
272 cli = next((i for i, line in enumerate(lines[:-1]) if '* changes\\:' in line), None)
273 first_msg = lines[cli+1] if cli else None
275 if self.verbose >= 1:
276 print("did not find Change-Id or commit-msg in %s, trying with merge commit-msg: %s"
277 % (project, first_msg))
278 gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
280 return ChangeId(gerrits[0]["id"], True)
282 print("did not find Change-Id for %s" % project)
284 return ChangeId(None, False)
286 def find_distro_changeid(self, project):
288 Find a distribution Change-Id by finding a project jar in
289 the distribution and parsing it's git.properties.
291 :param str project: The project to search
292 :return ChangeId: The Change-Id with a valid Change-Id or None if not found
294 project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project)
296 for root, dirs, files in os.walk(project_dir):
298 if file_.endswith(".jar"):
299 fullpath = os.path.join(root, file_)
300 pfile = self.extract_gitproperties_file(fullpath)
302 changeid = self.get_changeid_from_properties(project, pfile)
303 if changeid.changeid:
306 print("Could not find %s Change-Id in git.properties" % project)
307 break # all jars will have the same git.properties
308 if pfile is not None:
309 break # all jars will have the same git.properties
311 print("Could not find a git.properties file for %s" % project)
312 return ChangeId(None, False)
315 self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose)
316 self.set_projects(self.project_names)
318 def print_options(self):
319 print("Using these options: branch: %s, limit: %d, qlimit: %d"
320 % (self.branch, self.limit, self.qlimit))
321 print("remote_url: %s" % self.remote_url)
322 print("distro_path: %s" % self.distro_path)
323 print("projects: %s" % (", ".join(map(str, self.projects))))
324 print("gerrit 00 is the most recent patch from which the project was built followed by the next most"
325 " recently merged patches up to %s." % self.limit)
329 Internal wrapper between main, options parser and internal code.
331 Get the gerrit for the given Change-Id and parse it.
332 Loop over all projects:
333 get qlimit gerrits and parse them
334 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
336 # TODO: need method to validate the branch matches the distribution
341 if self.distro_url is not None:
342 self.download_distro()
344 for project in sorted(self.projects):
345 if self.verbose >= 1:
346 print("Processing %s" % project)
347 changeid = self.find_distro_changeid(project)
348 if changeid.changeid:
349 self.projects[project]['commit'] = changeid.changeid
350 self.projects[project]["includes"] =\
351 self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged)
355 parser = argparse.ArgumentParser(description=COPYRIGHT)
357 parser.add_argument("-b", "--branch", default=self.BRANCH,
358 help="git branch for patch under test")
359 parser.add_argument("-d", "--distro-path", dest="distro_path", default=self.DISTRO_PATH,
360 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH)
361 parser.add_argument("-u", "--distro-url", dest="distro_url", default=self.DISTRO_URL,
362 help="optional url to download a distribution " + str(self.DISTRO_URL))
363 parser.add_argument("-l", "--limit", dest="limit", type=int, default=self.LIMIT,
364 help="number of gerrits to return")
365 parser.add_argument("-p", "--projects", dest="projects", default=self.PROJECT_NAMES,
366 help="list of projects to include in output")
367 parser.add_argument("-q", "--query-limit", dest="qlimit", type=int, default=self.QUERY_LIMIT,
368 help="number of gerrits to search")
369 parser.add_argument("-r", "--remote", dest="remote_url", default=self.REMOTE_URL,
370 help="git remote url to use for gerrit")
371 parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=self.VERBOSE,
372 help="Output more information about what's going on")
373 parser.add_argument("--license", dest="license", action="store_true",
374 help="Print the license and exit")
375 parser.add_argument("-V", "--version", action="version",
376 version="%s version %s" %
377 (os.path.split(sys.argv[0])[-1], 0.1))
379 options = parser.parse_args()
385 self.branch = options.branch
386 self.distro_path = options.distro_path
387 self.distro_url = options.distro_url
388 self.limit = options.limit
389 self.qlimit = options.qlimit
390 self.remote_url = options.remote_url
391 self.verbose = options.verbose
392 if options.projects != self.PROJECT_NAMES:
393 self.project_names = options.projects.split(',')
395 # TODO: add check to verify that the remote can be reached,
396 # though the first gerrit query will fail anyways
398 projects = self.run_cmd()
399 self.pretty_print_projects(projects)
407 except Exception as e:
408 # If one does unguarded print(e) here, in certain locales the implicit
409 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
410 # range(128)". See rhbz#1058167.
414 # Python 3, we"re home free.
417 print(u.encode("utf-8"))
419 sys.exit(getattr(e, "EXIT_CODE", -1))
422 if __name__ == "__main__":