13 1. What about the time between when a merge is submitted and
14 the patch is in the distribution? Should we look at the other
15 events and see when the merge job finished?
16 2. Use the git query option to request records in multiple queries
17 rather than grabbing all 50 in one shot. Keep going until the requested
18 number is found. Verify if this is better than just doing all 50 in one
19 shot since multiple requests are ssh round trips per request.
22 # This file started as an exact copy of git-review so including it"s copyright
25 Copyright (C) 2011-2017 OpenStack LLC.
27 Licensed under the Apache License, Version 2.0 (the "License");
28 you may not use this file except in compliance with the License.
29 You may obtain a copy of the License at
31 http://www.apache.org/licenses/LICENSE-2.0
33 Unless required by applicable law or agreed to in writing, software
34 distributed under the License is distributed on an "AS IS" BASIS,
35 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
38 See the License for the specific language governing permissions and
39 limitations under the License."""
43 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
44 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
45 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
46 NETVIRT_PROJECTS = ["netvirt", "controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf",
47 "neutron", "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"]
48 PROJECT_NAMES = NETVIRT_PROJECTS
50 DISTRO_PATH = "/tmp/distribution-karaf"
52 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
58 distro_path = DISTRO_PATH
59 distro_url = DISTRO_URL
60 project_names = PROJECT_NAMES
64 remote_url = REMOTE_URL
68 def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH,
69 limit=LIMIT, qlimit=QUERY_LIMIT,
70 project_names=PROJECT_NAMES, remote_url=REMOTE_URL,
73 self.distro_path = distro_path
76 self.project_names = project_names
77 self.remote_url = remote_url
78 self.verbose = verbose
81 def pretty_print_gerrits(project, gerrits):
85 print("i grantedOn lastUpdatd chang subject")
86 print("-- ---------- ---------- ----- -----------------------------------------")
87 for i, gerrit in enumerate(gerrits):
88 if isinstance(gerrit, dict):
89 print("%02d %010d %010d %5s %s"
91 gerrit["grantedOn"] if "grantedOn" in gerrit else 0,
92 gerrit["lastUpdated"] if "lastUpdated" in gerrit else 0,
93 gerrit["number"] if "number" in gerrit else "00000",
94 gerrit["subject"] if "subject" in gerrit else "none"))
96 def pretty_print_projects(self, projects):
97 if isinstance(projects, dict):
98 for project_name, values in projects.items():
99 if "includes" in values:
100 self.pretty_print_gerrits(project_name, values["includes"])
102 def set_projects(self, project_names=PROJECT_NAMES):
103 for project in project_names:
104 self.projects[project] = {"commit": [], "includes": []}
106 def download_distro(self):
108 Download the distribution from self.distro_url and extract it to self.distro_path
110 if self.verbose >= 2:
111 print("attempting to download distribution from %s and extract to %s " %
112 (self.distro_url, self.distro_path))
114 tmp_distro_zip = '/tmp/distro.zip'
115 tmp_unzipped_location = '/tmp/distro_unzipped'
116 downloader = urllib3.PoolManager(cert_reqs='CERT_NONE')
118 # disabling warnings to prevent scaring the user with InsecureRequestWarning
119 urllib3.disable_warnings()
121 downloaded_distro = downloader.request('GET', self.distro_url)
122 with open(tmp_distro_zip, 'wb') as f:
123 f.write(downloaded_distro.data)
125 downloaded_distro.release_conn()
127 # after the .zip is extracted we want to rename it to be the distro_path which may have
128 # been given by the user
129 distro_zip = zipfile.ZipFile(tmp_distro_zip, 'r')
130 distro_zip.extractall(tmp_unzipped_location)
131 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
133 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
134 # there is relevant (and maybe already put there by this tool earlier)
136 os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path)
139 print("Unable to move extracted files from %s to %s. Using whatever bits are already there" %
140 (tmp_unzipped_location, self.distro_path))
142 def get_includes(self, project, changeid=None, msg=None):
144 Get the gerrits that would be included before the change merge time.
146 :param str project: The project to search
147 :param str changeid: The Change-Id of the gerrit to use for the merge time
148 :param str msg: The commit message of the gerrit to use for the merge time
149 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
151 includes = self.gerritquery.get_gerrits(project, changeid, 1, msg)
153 print("Review %s in %s:%s was not found" % (changeid, project, self.gerritquery.branch))
156 gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg)
157 for gerrit in gerrits:
158 # don"t include the same change in the list
159 if gerrit["id"] == changeid:
162 # TODO: should the check be < or <=?
163 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
164 includes.append(gerrit)
166 # break out if we have the number requested
167 if len(includes) == self.limit + 1:
170 if len(includes) != self.limit + 1:
171 print("%s query limit was not large enough to capture %d gerrits" % (project, self.limit))
176 def extract_gitproperties_file(fullpath):
178 Extract a git.properties from a jar archive.
180 :param str fullpath: Path to the jar
181 :return str: Containing git.properties or None if not found
183 if zipfile.is_zipfile(fullpath):
184 zf = zipfile.ZipFile(fullpath, "r")
186 pfile = zf.open("META-INF/git.properties")
192 def get_changeid_from_properties(self, project, pfile):
194 Parse the git.properties file to find a Change-Id.
196 There are a few different forms that we know of so far:
197 - I0123456789012345678901234567890123456789
199 - no Change-Id at all. There is a commit message and commit hash.
200 In this example the commit hash cannot be found because it was a merge
201 so you must use the message. Note spaces need to be replaced with +"s
203 :param str project: The project to search
204 :param str pfile: String containing the content of the git.properties file
205 :return str: The Change-Id or None if not found
207 # match a 40 or 8 char Change-Id hash. both start with I
208 regex = re.compile(r'\bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b')
209 changeid = regex.search(pfile)
211 return changeid.group()
213 # Didn"t find a Change-Id so try to get a commit message
214 # match on "blah" but only keep the blah
215 regex_msg = re.compile(r'"([^"]*)"|^git.commit.message.short=(.*)$')
216 msg = regex_msg.search(pfile)
217 if self.verbose >= 2:
218 print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group()))
221 # TODO: add new query using this msg
222 gerrits = self.gerritquery.get_gerrits(project, None, 1, msg.group())
224 return gerrits[0]["id"]
227 def find_distro_changeid(self, project):
229 Find a distribution Change-Id by finding a project jar in
230 the distribution and parsing it's git.properties.
232 :param str project: The project to search
233 :return str: The Change-Id or None if not found
235 project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project)
237 for root, dirs, files in os.walk(project_dir):
239 if file_.endswith(".jar"):
240 fullpath = os.path.join(root, file_)
241 pfile = self.extract_gitproperties_file(fullpath)
243 changeid = self.get_changeid_from_properties(project, pfile)
247 print("Could not find %s Change-Id in git.properties" % project)
248 break # all jars will have the same git.properties
249 if pfile is not None:
250 break # all jars will have the same git.properties
254 self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose)
255 self.set_projects(self.project_names)
257 def print_options(self):
258 print("Using these options: branch: %s, limit: %d, qlimit: %d"
259 % (self.branch, self.limit, self.qlimit))
260 print("remote_url: %s" % self.remote_url)
261 print("distro_path: %s" % self.distro_path)
262 print("projects: %s" % (", ".join(map(str, self.projects))))
263 print("gerrit 00 is the most recent patch from which the project was built followed by the next most"
264 " recently merged patches up to %s." % self.limit)
268 Internal wrapper between main, options parser and internal code.
270 Get the gerrit for the given Change-Id and parse it.
271 Loop over all projects:
272 get qlimit gerrits and parse them
273 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
275 # TODO: need method to validate the branch matches the distribution
280 if self.distro_url is not None:
281 self.download_distro()
283 for project in self.projects:
284 changeid = self.find_distro_changeid(project)
286 self.projects[project]["includes"] = self.get_includes(project, changeid)
290 parser = argparse.ArgumentParser(description=COPYRIGHT)
292 parser.add_argument("-b", "--branch", default=self.BRANCH,
293 help="git branch for patch under test")
294 parser.add_argument("-d", "--distro-path", dest="distro_path", default=self.DISTRO_PATH,
295 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH)
296 parser.add_argument("-u", "--distro-url", dest="distro_url", default=self.DISTRO_URL,
297 help="optional url to download a distribution " + str(self.DISTRO_URL))
298 parser.add_argument("-l", "--limit", dest="limit", type=int, default=self.LIMIT,
299 help="number of gerrits to return")
300 parser.add_argument("-p", "--projects", dest="projects", default=self.PROJECT_NAMES,
301 help="list of projects to include in output")
302 parser.add_argument("-q", "--query-limit", dest="qlimit", type=int, default=self.QUERY_LIMIT,
303 help="number of gerrits to search")
304 parser.add_argument("-r", "--remote", dest="remote_url", default=self.REMOTE_URL,
305 help="git remote url to use for gerrit")
306 parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=self.VERBOSE,
307 help="Output more information about what's going on")
308 parser.add_argument("--license", dest="license", action="store_true",
309 help="Print the license and exit")
310 parser.add_argument("-V", "--version", action="version",
311 version="%s version %s" %
312 (os.path.split(sys.argv[0])[-1], 0.1))
314 options = parser.parse_args()
320 self.branch = options.branch
321 self.distro_path = options.distro_path
322 self.distro_url = options.distro_url
323 self.limit = options.limit
324 self.qlimit = options.qlimit
325 self.remote_url = options.remote_url
326 self.verbose = options.verbose
327 if options.projects != self.PROJECT_NAMES:
328 self.project_names = options.projects.split(',')
330 # TODO: add check to verify that the remote can be reached,
331 # though the first gerrit query will fail anyways
333 projects = self.run_cmd()
334 self.pretty_print_projects(projects)
342 except Exception as e:
343 # If one does unguarded print(e) here, in certain locales the implicit
344 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
345 # range(128)". See rhbz#1058167.
349 # Python 3, we"re home free.
352 print(u.encode("utf-8"))
354 sys.exit(getattr(e, "EXIT_CODE", -1))
357 if __name__ == "__main__":