13 1. What about the time between when a merge is submitted and
14 the patch is in the distribution? Should we look at the other
15 events and see when the merge job finished?
16 2. Use the git query option to request records in multiple queries
17 rather than grabbing all 50 in one shot. Keep going until the requested
18 number is found. Verify if this is better than just doing all 50 in one
19 shot since multiple requests are ssh round trips per request.
22 # This file started as an exact copy of git-review so including it"s copyright
25 Copyright (C) 2011-2017 OpenStack LLC.
27 Licensed under the Apache License, Version 2.0 (the "License");
28 you may not use this file except in compliance with the License.
29 You may obtain a copy of the License at
31 http://www.apache.org/licenses/LICENSE-2.0
33 Unless required by applicable law or agreed to in writing, software
34 distributed under the License is distributed on an "AS IS" BASIS,
35 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
38 See the License for the specific language governing permissions and
39 limitations under the License."""
43 # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
44 # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
45 # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
46 NETVIRT_PROJECTS = ["controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf", "neutron",
47 "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"]
48 PROJECT_NAMES = NETVIRT_PROJECTS
50 DISTRO_PATH = "/tmp/distribution-karaf"
52 REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
58 distro_path = DISTRO_PATH
59 distro_url = DISTRO_URL
60 project_names = PROJECT_NAMES
64 remote_url = REMOTE_URL
68 def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH,
69 limit=LIMIT, qlimit=QUERY_LIMIT,
70 project_names=PROJECT_NAMES, remote_url=REMOTE_URL,
73 self.distro_path = distro_path
76 self.project_names = project_names
77 self.remote_url = remote_url
78 self.verbose = verbose
79 self.set_projects(project_names)
82 def pretty_print_gerrits(project, gerrits):
86 print("i grantedOn lastUpdatd chang subject")
87 print("-- ---------- ---------- ----- -----------------------------------------")
88 for i, gerrit in enumerate(gerrits):
89 print("%02d %d %d %s %s" % (i, gerrit["grantedOn"], gerrit["lastUpdated"],
90 gerrit["number"], gerrit["subject"]))
92 def pretty_print_includes(self, includes):
93 for project, gerrits in includes.items():
94 self.pretty_print_gerrits(project, gerrits)
96 def pretty_print_projects(self, projects):
97 for project_name, values in projects.items():
98 self.pretty_print_gerrits(project_name, values["includes"])
100 def set_projects(self, project_names=PROJECT_NAMES):
101 for project in project_names:
102 self.projects[project] = {"commit": [], "includes": []}
104 def download_distro(self):
106 Download the distribution from self.distro_url and extract it to self.distro_path
108 if self.verbose >= 2:
109 print("attempting to download distribution from %s and extract to %s " %
110 (self.distro_url, self.distro_path))
112 tmp_distro_zip = '/tmp/distro.zip'
113 tmp_unzipped_location = '/tmp/distro_unzipped'
114 downloader = urllib3.PoolManager(cert_reqs='CERT_NONE')
116 # disabling warnings to prevent scaring the user with InsecureRequestWarning
117 urllib3.disable_warnings()
119 downloaded_distro = downloader.request('GET', self.distro_url)
120 with open(tmp_distro_zip, 'wb') as f:
121 f.write(downloaded_distro.data)
123 downloaded_distro.release_conn()
125 # after the .zip is extracted we want to rename it to be the distro_path which may have
126 # been given by the user
127 distro_zip = zipfile.ZipFile(tmp_distro_zip, 'r')
128 distro_zip.extractall(tmp_unzipped_location)
129 unzipped_distro_folder = os.listdir(tmp_unzipped_location)
131 # if the distro_path already exists, we wont overwrite it and just continue hoping what's
132 # there is relevant (and maybe already put there by this tool earlier)
134 os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path)
137 print("Unable to move extracted files from %s to %s. Using whatever bits are already there" %
138 (tmp_unzipped_location, self.distro_path))
140 def get_includes(self, project, changeid=None, msg=None):
142 Get the gerrits that would be included before the change merge time.
144 :param str project: The project to search
145 :param str changeid: The Change-Id of the gerrit to use for the merge time
146 :param str msg: The commit message of the gerrit to use for the merge time
147 :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
149 includes = self.gerritquery.get_gerrits(project, changeid, 1, msg)
151 print("Review %s in %s:%s was not found" % (changeid, project, self.gerritquery.branch))
154 gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg)
155 for gerrit in gerrits:
156 # don"t include the same change in the list
157 if gerrit["id"] == changeid:
160 # TODO: should the check be < or <=?
161 if gerrit["grantedOn"] <= includes[0]["grantedOn"]:
162 includes.append(gerrit)
164 # break out if we have the number requested
165 if len(includes) == self.limit + 1:
168 if len(includes) != self.limit + 1:
169 print("%s query limit was not large enough to capture %d gerrits" % (project, self.limit))
174 def extract_gitproperties_file(fullpath):
176 Extract a git.properties from a jar archive.
178 :param str fullpath: Path to the jar
179 :return str: Containing git.properties or None if not found
181 if zipfile.is_zipfile(fullpath):
182 zf = zipfile.ZipFile(fullpath, "r")
184 pfile = zf.open("META-INF/git.properties")
190 def get_changeid_from_properties(self, project, pfile):
192 Parse the git.properties file to find a Change-Id.
194 There are a few different forms that we know of so far:
195 - I0123456789012345678901234567890123456789
197 - no Change-Id at all. There is a commit message and commit hash.
198 In this example the commit hash cannot be found because it was a merge
199 so you must use the message. Note spaces need to be replaced with +"s
201 :param str project: The project to search
202 :param str pfile: String containing the content of the git.properties file
203 :return str: The Change-Id or None if not found
205 # match a 40 or 8 char Change-Id hash. both start with I
206 regex = re.compile(r'\bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b')
207 changeid = regex.search(pfile)
209 return changeid.group()
211 # Didn"t find a Change-Id so try to get a commit message
212 # match on "blah" but only keep the blah
213 regex_msg = re.compile(r'"([^"]*)"|^git.commit.message.short=(.*)$')
214 msg = regex_msg.search(pfile)
215 if self.verbose >= 2:
216 print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group()))
219 # TODO: add new query using this msg
220 gerrits = self.gerritquery.get_gerrits(project, None, 1, msg.group())
222 return gerrits[0]["id"]
225 def find_distro_changeid(self, project):
227 Find a distribution Change-Id by finding a project jar in
228 the distribution and parsing it's git.properties.
230 :param str project: The project to search
231 :return str: The Change-Id or None if not found
233 project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project)
235 for root, dirs, files in os.walk(project_dir):
237 if file_.endswith(".jar"):
238 fullpath = os.path.join(root, file_)
239 pfile = self.extract_gitproperties_file(fullpath)
241 changeid = self.get_changeid_from_properties(project, pfile)
245 print("Could not find %s Change-Id in git.properties" % project)
246 break # all jars will have the same git.properties
247 if pfile is not None:
248 break # all jars will have the same git.properties
252 self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose)
253 self.set_projects(self.project_names)
255 def print_options(self):
256 print("Using these options: branch: %s, limit: %d, qlimit: %d"
257 % (self.branch, self.limit, self.qlimit))
258 print("remote_url: %s" % self.remote_url)
259 print("distro_path: %s" % self.distro_path)
260 print("projects: %s" % (", ".join(map(str, self.projects))))
261 print("gerrit 00 is the most recent patch from which the project was built followed by the next most"
262 " recently merged patches up to %s." % self.limit)
266 Internal wrapper between main, options parser and internal code.
268 Get the gerrit for the given Change-Id and parse it.
269 Loop over all projects:
270 get qlimit gerrits and parse them
271 copy up to limit gerrits with a SUBM time (grantedOn) <= to the given change-id
273 # TODO: need method to validate the branch matches the distribution
278 if self.distro_url is not None:
279 self.download_distro()
281 for project in self.projects:
282 changeid = self.find_distro_changeid(project)
284 self.projects[project]["includes"] = self.get_includes(project, changeid)
288 parser = argparse.ArgumentParser(description=COPYRIGHT)
290 parser.add_argument("-b", "--branch", default=self.BRANCH,
291 help="git branch for patch under test")
292 parser.add_argument("-d", "--distro-path", dest="distro_path", default=self.DISTRO_PATH,
293 help="path to the expanded distribution, i.e. " + self.DISTRO_PATH)
294 parser.add_argument("-u", "--distro-url", dest="distro_url", default=self.DISTRO_URL,
295 help="optional url to download a distribution " + str(self.DISTRO_URL))
296 parser.add_argument("-l", "--limit", dest="limit", type=int, default=self.LIMIT,
297 help="number of gerrits to return")
298 parser.add_argument("-p", "--projects", dest="projects", default=self.PROJECT_NAMES,
299 help="list of projects to include in output")
300 parser.add_argument("-q", "--query-limit", dest="qlimit", type=int, default=self.QUERY_LIMIT,
301 help="number of gerrits to search")
302 parser.add_argument("-r", "--remote", dest="remote_url", default=self.REMOTE_URL,
303 help="git remote url to use for gerrit")
304 parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=self.VERBOSE,
305 help="Output more information about what's going on")
306 parser.add_argument("--license", dest="license", action="store_true",
307 help="Print the license and exit")
308 parser.add_argument("-V", "--version", action="version",
309 version="%s version %s" %
310 (os.path.split(sys.argv[0])[-1], 0.1))
312 options = parser.parse_args()
318 self.branch = options.branch
319 self.distro_path = options.distro_path
320 self.distro_url = options.distro_url
321 self.limit = options.limit
322 self.project_names = options.projects
323 self.qlimit = options.qlimit
324 self.remote_url = options.remote_url
325 self.verbose = options.verbose
327 # TODO: add check to verify that the remote can be reached,
328 # though the first gerrit query will fail anyways
330 projects = self.run_cmd()
331 self.pretty_print_projects(projects)
339 except Exception as e:
340 # If one does unguarded print(e) here, in certain locales the implicit
341 # str(e) blows up with familiar "UnicodeEncodeError ... ordinal not in
342 # range(128)". See rhbz#1058167.
346 # Python 3, we"re home free.
349 print(u.encode("utf-8"))
351 sys.exit(getattr(e, "EXIT_CODE", -1))
354 if __name__ == "__main__":