2 This module contains functions to manipulate gerrit queries.
13 # TODO: Haven't tested python 3
18 urlencode = urllib.urlencode
19 urljoin = urlparse.urljoin
20 urlparse = urlparse.urlparse
26 urlencode = urllib.parse.urlencode
27 urljoin = urllib.parse.urljoin
28 urlparse = urllib.parse.urlparse
32 class GitReviewException(Exception):
36 class CommandFailed(GitReviewException):
37 """Command Failure Analysis"""
39 def __init__(self, *args):
40 Exception.__init__(self, *args)
41 (self.rc, self.output, self.argv, self.envp) = args
42 self.quickmsg = dict([
43 ("argv", " ".join(self.argv)),
45 ("output", self.output)])
48 return self.__doc__ + """
49 The following command failed with exit code %(rc)d
51 -----------------------
53 -----------------------""" % self.quickmsg
57 REMOTE_URL = 'ssh://git.opendaylight.org:29418'
61 remote_url = REMOTE_URL
63 query_limit = QUERY_LIMIT
66 def __init__(self, remote_url, branch, query_limit, verbose):
67 self.remote_url = remote_url
69 self.query_limit = query_limit
70 self.verbose = verbose
72 def set_verbose(self, verbose):
73 self.verbose = verbose
76 def print_safe_encoding(string):
77 if sys.stdout.encoding is None:
78 # just print(string) could still throw a UnicodeEncodeError sometimes so casting string to unicode
79 print(unicode(string))
81 print(string.encode(sys.stdout.encoding, 'replace'))
83 def run_command_status(self, *argv, **kwargs):
86 print(datetime.datetime.now(), "Running:", " ".join(argv))
88 # for python2 compatibility with shlex
89 if sys.version_info < (3,) and isinstance(argv[0], unicode):
90 argv = shlex.split(argv[0].encode('utf-8'))
92 argv = shlex.split(str(argv[0]))
93 stdin = kwargs.pop('stdin', None)
94 newenv = os.environ.copy()
96 newenv['LANGUAGE'] = 'C'
98 p = subprocess.Popen(argv,
99 stdin=subprocess.PIPE if stdin else None,
100 stdout=subprocess.PIPE,
101 stderr=subprocess.STDOUT,
103 (out, nothing) = p.communicate(stdin)
104 out = out.decode('utf-8', 'replace')
105 return p.returncode, out.strip()
107 def run_command(self, *argv, **kwargs):
108 (rc, output) = self.run_command_status(*argv, **kwargs)
111 def run_command_exc(self, klazz, *argv, **env):
113 Run command *argv, on failure raise klazz
115 klazz should be derived from CommandFailed
117 (rc, output) = self.run_command_status(*argv, **env)
119 raise klazz(rc, output, argv, env)
122 def parse_gerrit_ssh_params_from_git_url(self):
124 Parse a given Git "URL" into Gerrit parameters. Git "URLs" are either
125 real URLs or SCP-style addresses.
128 # The exact code for this in Git itself is a bit obtuse, so just do
129 # something sensible and pythonic here instead of copying the exact
132 # Handle real(ish) URLs
133 if "://" in self.remote_url:
134 parsed_url = urlparse(self.remote_url)
135 path = parsed_url.path
137 hostname = parsed_url.netloc
139 port = parsed_url.port
141 # Workaround bug in urlparse on OSX
142 if parsed_url.scheme == "ssh" and parsed_url.path[:2] == "//":
143 hostname = parsed_url.path[2:].split("/")[0]
146 (username, hostname) = hostname.split("@")
148 (hostname, port) = hostname.split(":")
153 # Handle SCP-style addresses
157 (hostname, path) = self.remote_url.split(":", 1)
159 (username, hostname) = hostname.split("@", 1)
161 # Strip leading slash and trailing .git from the path to form the project
163 project_name = re.sub(r"^/|(\.git$)", "", path)
165 return hostname, username, port, project_name
167 def gerrit_request(self, request):
169 Send a gerrit request and receive a response.
171 :param str request: A gerrit query
172 :return unicode: The JSON response
174 (hostname, username, port, project_name) = \
175 self.parse_gerrit_ssh_params_from_git_url()
177 port_data = "p%s" % port if port is not None else ""
181 userhost = "%s@%s" % (username, hostname)
183 if self.verbose >= 2:
184 print("gerrit request %s %s" % (self.remote_url, request))
185 output = self.run_command_exc(
187 "ssh", "-x" + port_data, userhost,
189 if self.verbose >= 3:
190 self.print_safe_encoding(output)
193 def make_gerrit_query(self, project, changeid=None, limit=1, msg=None, status=None, comments=False):
195 Make a gerrit query by combining the given options.
197 :param str project: The project to search
198 :param str changeid: A Change-Id to search
199 :param int limit: The number of items to return
200 :param str msg or None: A commit-msg to search
201 :param str status or None: The gerrit status, i.e. merged
202 :param bool comments: If true include comments
203 :return str: A gerrit query
205 query = "gerrit query --format=json limit:%d " \
206 "project:%s branch:%s" \
207 % (limit, project, self.branch)
209 query += " change:%s" % changeid
211 query += " message:%s" % msg
213 query += " status:%s --all-approvals" % status
215 query += " --comments"
218 def parse_gerrit(self, line, parse_exc=Exception):
220 Parse a single gerrit line and copy certain fields to a dictionary.
222 The merge time is found by looking for the Patch Set->Approval with
223 a SUBM type. Then use the grantedOn value.
225 :param str line: A single line from a previous gerrit query
226 :param parse_exc: The exception to except
227 :return dict: Pairs of gerrit items and their values
231 if line and line[0] == "{":
233 data = json.loads(line)
234 parsed['id'] = data['id']
235 parsed['number'] = data['number']
236 parsed['subject'] = data['subject']
237 parsed['url'] = data['url']
238 parsed['lastUpdated'] = data['lastUpdated']
239 parsed['grantedOn'] = 0
240 if "patchSets" in data:
241 patch_sets = data['patchSets']
242 for patch_set in reversed(patch_sets):
243 if "approvals" in patch_set:
244 approvals = patch_set['approvals']
245 for approval in approvals:
246 if 'type' in approval and approval['type'] == 'SUBM':
247 parsed['grantedOn'] = approval['grantedOn']
249 if parsed['grantedOn'] != 0:
251 if "comments" in data:
252 comments = data['comments']
253 for comment in reversed(comments):
254 if "message" in comment and "timestamp" in comment:
255 message = comment['message']
256 timestamp = comment['timestamp']
257 if "Build Started" in message and "patch-test" in message:
258 parsed['grantedOn'] = timestamp
262 print("Failed to decode JSON: %s" % traceback.format_exc())
263 self.print_safe_encoding(line)
264 except Exception as err:
265 print("Exception: %s" % traceback.format_exc())
269 def extract_lines_from_json(self, changes):
271 Extract a list of lines from the JSON gerrit query response.
275 :param unicode changes: The full JSON gerrit query response
276 :return list: Lines of the JSON
279 for line in changes.split("\n"):
280 if line.find('"type":"error","message"') != -1:
281 print("there was a query error")
283 if line.find('stats') == -1:
285 if self.verbose >= 2:
286 print("get_gerrit_lines: found %d lines" % len(lines))
289 def get_gerrits(self, project, changeid=None, limit=1, msg=None, status=None, comments=False):
291 Get a list of gerrits from gerrit query request.
293 Gerrit returns queries in order of lastUpdated so resort based on merge time.
294 Also because gerrit returns them in lastUpdated order, it means all gerrits
295 merged after the one we are using will be returned, so the query limit needs to be
296 high enough to capture those extra merges plus the limit requested.
297 TODO: possibly add the before query to set a start time for the query around the change
299 :param str project: The project to search
300 :param str or None changeid: A Change-Id to search
301 :param int limit: The number of items to return
302 :param str or None msg: A commit-msg to search
303 :param str or None status: The gerrit status, i.e. merged
304 :param bool comments: If true include comments
305 :return str: List of gerrits sorted by merge time
307 query = self.make_gerrit_query(project, changeid, limit, msg, status, comments)
308 changes = self.gerrit_request(query)
309 lines = self.extract_lines_from_json(changes)
312 gerrits.append(self.parse_gerrit(line))
314 from operator import itemgetter
315 sorted_gerrits = sorted(gerrits, key=itemgetter('grantedOn'), reverse=True)
316 return sorted_gerrits