2 This module contains functions to manipulate gerrit queries.
14 # TODO: Haven't tested python 3
19 urlencode = urllib.urlencode
20 urljoin = urlparse.urljoin
21 urlparse = urlparse.urlparse
27 urlencode = urllib.parse.urlencode
28 urljoin = urllib.parse.urljoin
29 urlparse = urllib.parse.urlparse
33 logger = logging.getLogger("changes.gerritquery")
36 class GitReviewException(Exception):
40 class CommandFailed(GitReviewException):
41 """Command Failure Analysis"""
43 def __init__(self, *args):
44 Exception.__init__(self, *args)
45 (self.rc, self.output, self.argv, self.envp) = args
47 [("argv", " ".join(self.argv)), ("rc", self.rc), ("output", self.output)]
54 The following command failed with exit code %(rc)d
56 -----------------------
58 -----------------------"""
64 REMOTE_URL = "ssh://git.opendaylight.org:29418"
68 remote_url = REMOTE_URL
70 query_limit = QUERY_LIMIT
72 def __init__(self, remote_url, branch, query_limit, verbose):
73 self.remote_url = remote_url
75 self.query_limit = query_limit
76 self.verbose = verbose
79 def print_safe_encoding(string):
81 if type(string) == unicode:
83 if hasattr(sys.stdout, "encoding") and sys.stdout.encoding:
84 encoding = sys.stdout.encoding
85 return string.encode(encoding or "utf-8", "replace")
91 def run_command_status(self, *argv, **kwargs):
92 logger.debug("%s Running: %s", datetime.datetime.now(), " ".join(argv))
94 # for python2 compatibility with shlex
95 if sys.version_info < (3,) and isinstance(argv[0], unicode):
96 argv = shlex.split(argv[0].encode("utf-8"))
98 argv = shlex.split(str(argv[0]))
99 stdin = kwargs.pop("stdin", None)
100 newenv = os.environ.copy()
102 newenv["LANGUAGE"] = "C"
103 newenv.update(kwargs)
104 p = subprocess.Popen(
106 stdin=subprocess.PIPE if stdin else None,
107 stdout=subprocess.PIPE,
108 stderr=subprocess.STDOUT,
111 (out, nothing) = p.communicate(stdin)
112 out = out.decode("utf-8", "replace")
113 return p.returncode, out.strip()
115 def run_command(self, *argv, **kwargs):
116 (rc, output) = self.run_command_status(*argv, **kwargs)
119 def run_command_exc(self, klazz, *argv, **env):
121 Run command *argv, on failure raise klazz
123 klazz should be derived from CommandFailed
125 (rc, output) = self.run_command_status(*argv, **env)
127 raise klazz(rc, output, argv, env)
130 def parse_gerrit_ssh_params_from_git_url(self):
132 Parse a given Git "URL" into Gerrit parameters. Git "URLs" are either
133 real URLs or SCP-style addresses.
136 # The exact code for this in Git itself is a bit obtuse, so just do
137 # something sensible and pythonic here instead of copying the exact
140 # Handle real(ish) URLs
141 if "://" in self.remote_url:
142 parsed_url = urlparse(self.remote_url)
143 path = parsed_url.path
145 hostname = parsed_url.netloc
147 port = parsed_url.port
149 # Workaround bug in urlparse on OSX
150 if parsed_url.scheme == "ssh" and parsed_url.path[:2] == "//":
151 hostname = parsed_url.path[2:].split("/")[0]
154 (username, hostname) = hostname.split("@")
156 (hostname, port) = hostname.split(":")
161 # Handle SCP-style addresses
165 (hostname, path) = self.remote_url.split(":", 1)
167 (username, hostname) = hostname.split("@", 1)
169 # Strip leading slash and trailing .git from the path to form the project
171 project_name = re.sub(r"^/|(\.git$)", "", path)
173 return hostname, username, port, project_name
175 def gerrit_request(self, request):
177 Send a gerrit request and receive a response.
179 :param str request: A gerrit query
180 :return unicode: The JSON response
187 ) = self.parse_gerrit_ssh_params_from_git_url()
189 port_data = "p%s" % port if port is not None else ""
193 userhost = "%s@%s" % (username, hostname)
195 logger.debug("gerrit request %s %s" % (self.remote_url, request))
196 output = self.run_command_exc(
197 CommandFailed, "ssh", "-x" + port_data, userhost, request
199 if logger.isEnabledFor(logging.DEBUG):
200 logger.debug("%s", self.print_safe_encoding(output))
203 def make_gerrit_query(
214 Make a gerrit query by combining the given options.
216 :param str project: The project to search
217 :param str changeid: A Change-Id to search
218 :param int limit: The number of items to return
219 :param str msg or None: A commit-msg to search
220 :param str status or None: The gerrit status, i.e. merged
221 :param bool comments: If true include comments
222 :param commitid: A commit hash to search
223 :return str: A gerrit query
226 if project == "odlparent" or project == "yangtools":
227 query = "gerrit query --format=json limit:%d " "project:%s" % (
232 query = "gerrit query --format=json limit:%d " "project:%s branch:%s" % (
238 query += " change:%s" % changeid
240 query += " message:{%s}" % msg
242 query += " commit:%s" % commitid
244 query += " status:%s --all-approvals" % status
246 query += " --comments"
249 def parse_gerrit(self, line, parse_exc=Exception):
251 Parse a single gerrit line and copy certain fields to a dictionary.
253 The merge time is found by looking for the Patch Set->Approval with
254 a SUBM type. Then use the grantedOn value.
256 :param str line: A single line from a previous gerrit query
257 :param parse_exc: The exception to except
258 :return dict: Pairs of gerrit items and their values
262 if line and line[0] == "{":
264 data = json.loads(line)
265 parsed["id"] = data["id"]
266 parsed["number"] = data["number"]
267 parsed["subject"] = data["subject"]
268 parsed["url"] = data["url"]
269 parsed["lastUpdated"] = data["lastUpdated"]
270 parsed["grantedOn"] = 0
271 if "patchSets" in data:
272 patch_sets = data["patchSets"]
273 for patch_set in reversed(patch_sets):
274 if "approvals" in patch_set:
275 approvals = patch_set["approvals"]
276 for approval in approvals:
279 and approval["type"] == "SUBM"
281 parsed["grantedOn"] = approval["grantedOn"]
283 if parsed["grantedOn"] != 0:
285 if "comments" in data:
286 comments = data["comments"]
287 for comment in reversed(comments):
288 if "message" in comment and "timestamp" in comment:
289 message = comment["message"]
290 timestamp = comment["timestamp"]
292 "Build Started" in message
293 and "patch-test" in message
295 parsed["grantedOn"] = timestamp
298 logger.warn("Failed to decode JSON: %s", traceback.format_exc())
299 if logger.isEnabledFor(logging.DEBUG):
300 logger.warn(self.print_safe_encoding(line))
301 except Exception as err:
302 logger.warn("Exception: %s", traceback.format_exc())
306 def extract_lines_from_json(self, changes):
308 Extract a list of lines from the JSON gerrit query response.
312 :param unicode changes: The full JSON gerrit query response
313 :return list: Lines of the JSON
317 for i, line in enumerate(changes.split("\n")):
318 if line.find('"grantedOn":') != -1:
321 logger.debug("skipping: {}".format(line))
324 "get_gerrit_lines: found {} lines, skipped: {}".format(len(lines), skipped)
339 Get a list of gerrits from gerrit query request.
341 Gerrit returns queries in order of lastUpdated so resort based on merge time.
342 Also because gerrit returns them in lastUpdated order, it means all gerrits
343 merged after the one we are using will be returned, so the query limit needs to be
344 high enough to capture those extra merges plus the limit requested.
345 TODO: possibly add the before query to set a start time for the query around the change
347 :param str project: The project to search
348 :param str or None changeid: A Change-Id to search
349 :param int limit: The number of items to return
350 :param str or None msg: A commit-msg to search
351 :param str or None status: The gerrit status, i.e. merged
352 :param bool comments: If true include comments
353 :param commitid: A commit hash to search
354 :return str: List of gerrits sorted by merge time
357 "get_gerrits: project: %s, changeid: %s, limit: %d, msg: %s, status: %s, comments: %s, "
367 query = self.make_gerrit_query(
368 project, changeid, limit, msg, status, comments, commitid
370 changes = self.gerrit_request(query)
371 lines = self.extract_lines_from_json(changes)
375 gerrits.append(self.parse_gerrit(line))
377 from operator import itemgetter
380 logger.warn("No gerrits were found for %s", project)
383 sorted_gerrits = sorted(gerrits, key=itemgetter("grantedOn"), reverse=True)
384 except KeyError as e:
385 logger.warn("KeyError exception in %s, %s", project, str(e))
386 return sorted_gerrits