2 This module contains functions to manipulate gerrit queries.
14 # TODO: Haven't tested python 3
18 urlencode = urllib.urlencode
19 urljoin = urlparse.urljoin
20 urlparse = urlparse.urlparse
25 urlencode = urllib.parse.urlencode
26 urljoin = urllib.parse.urljoin
27 urlparse = urllib.parse.urlparse
31 class GitReviewException(Exception):
35 class CommandFailed(GitReviewException):
36 """Command Failure Analysis"""
38 def __init__(self, *args):
39 Exception.__init__(self, *args)
40 (self.rc, self.output, self.argv, self.envp) = args
41 self.quickmsg = dict([
42 ("argv", " ".join(self.argv)),
44 ("output", self.output)])
47 return self.__doc__ + """
48 The following command failed with exit code %(rc)d
50 -----------------------
52 -----------------------""" % self.quickmsg
56 REMOTE_URL = 'ssh://git.opendaylight.org:29418'
60 remote_url = REMOTE_URL
62 query_limit = QUERY_LIMIT
65 def __init__(self, remote_url, branch, query_limit, verbose):
66 self.remote_url = remote_url
68 self.query_limit = query_limit
69 self.verbose = verbose
71 def set_verbose(self, verbose):
72 self.verbose = verbose
75 def print_safe_encoding(string):
76 if sys.stdout.encoding is None:
77 # just print(string) could still throw a UnicodeEncodeError sometimes so casting string to unicode
78 print(unicode(string))
80 print(string.encode(sys.stdout.encoding, 'replace'))
82 def run_command_status(self, *argv, **kwargs):
85 print(datetime.datetime.now(), "Running:", " ".join(argv))
87 # for python2 compatibility with shlex
88 if sys.version_info < (3,) and isinstance(argv[0], unicode):
89 argv = shlex.split(argv[0].encode('utf-8'))
91 argv = shlex.split(str(argv[0]))
92 stdin = kwargs.pop('stdin', None)
93 newenv = os.environ.copy()
95 newenv['LANGUAGE'] = 'C'
97 p = subprocess.Popen(argv,
98 stdin=subprocess.PIPE if stdin else None,
99 stdout=subprocess.PIPE,
100 stderr=subprocess.STDOUT,
102 (out, nothing) = p.communicate(stdin)
103 out = out.decode('utf-8', 'replace')
104 return p.returncode, out.strip()
106 def run_command(self, *argv, **kwargs):
107 (rc, output) = self.run_command_status(*argv, **kwargs)
110 def run_command_exc(self, klazz, *argv, **env):
112 Run command *argv, on failure raise klazz
114 klazz should be derived from CommandFailed
116 (rc, output) = self.run_command_status(*argv, **env)
118 raise klazz(rc, output, argv, env)
121 def parse_gerrit_ssh_params_from_git_url(self):
123 Parse a given Git "URL" into Gerrit parameters. Git "URLs" are either
124 real URLs or SCP-style addresses.
127 # The exact code for this in Git itself is a bit obtuse, so just do
128 # something sensible and pythonic here instead of copying the exact
131 # Handle real(ish) URLs
132 if "://" in self.remote_url:
133 parsed_url = urlparse(self.remote_url)
134 path = parsed_url.path
136 hostname = parsed_url.netloc
138 port = parsed_url.port
140 # Workaround bug in urlparse on OSX
141 if parsed_url.scheme == "ssh" and parsed_url.path[:2] == "//":
142 hostname = parsed_url.path[2:].split("/")[0]
145 (username, hostname) = hostname.split("@")
147 (hostname, port) = hostname.split(":")
152 # Handle SCP-style addresses
156 (hostname, path) = self.remote_url.split(":", 1)
158 (username, hostname) = hostname.split("@", 1)
160 # Strip leading slash and trailing .git from the path to form the project
162 project_name = re.sub(r"^/|(\.git$)", "", path)
164 return hostname, username, port, project_name
166 def gerrit_request(self, request):
168 Send a gerrit request and receive a response.
170 :param str request: A gerrit query
171 :return unicode: The JSON response
173 (hostname, username, port, project_name) = \
174 self.parse_gerrit_ssh_params_from_git_url()
176 port_data = "p%s" % port if port is not None else ""
180 userhost = "%s@%s" % (username, hostname)
182 if self.verbose >= 2:
183 print("gerrit request %s %s" % (self.remote_url, request))
184 output = self.run_command_exc(
186 "ssh", "-x" + port_data, userhost,
188 if self.verbose >= 3:
189 self.print_safe_encoding(output)
192 def make_gerrit_query(self, project, changeid=None, limit=1, msg=None):
194 Make a gerrit query by combining the given options.
196 :param str project: The project to search
197 :param str changeid: A Change-Id to search
198 :param int limit: The number of items to return
199 :param str msg: A commit-msg to search
200 :return str: A gerrit query
202 query = "gerrit query --format=json limit:%d status:merged --all-approvals " \
203 "project:%s branch:%s" \
204 % (limit, project, self.branch)
206 query += " change:%s" % changeid
208 query += " message:%s" % msg
211 def parse_gerrit(self, line, parse_exc=Exception):
213 Parse a single gerrit line and copy certain fields to a dictionary.
215 The merge time is found by looking for the Patch Set->Approval with
216 a SUBM type. Then use the grantedOn value.
218 :param str line: A single line from a previous gerrit query
219 :param parse_exc: The exception to except
220 :return dict: Pairs of gerrit items and their values
224 if line and line[0] == "{":
226 data = json.loads(line)
227 parsed['id'] = data['id']
228 parsed['number'] = data['number']
229 parsed['subject'] = data['subject']
230 parsed['url'] = data['url']
231 parsed['lastUpdated'] = data['lastUpdated']
232 if "patchSets" in data:
233 patch_sets = data['patchSets']
234 for patch_set in reversed(patch_sets):
235 if "approvals" in patch_set:
236 approvals = patch_set['approvals']
237 for approval in approvals:
238 if 'type' in approval and approval['type'] == 'SUBM':
239 parsed['grantedOn'] = approval['grantedOn']
241 if parsed['grantedOn']:
245 print("Failed to decode JSON: %s" % traceback.format_exc())
246 self.print_safe_encoding(line)
247 except Exception as err:
248 print("Exception: %s" % traceback.format_exc())
252 def extract_lines_from_json(self, changes):
254 Extract a list of lines from the JSON gerrit query response.
258 :param unicode changes: The full JSON gerrit query response
259 :return list: Lines of the JSON
262 for line in changes.split("\n"):
263 if line.find('"type":"error","message"') != -1:
264 print("there was a query error")
266 if line.find('stats') == -1:
268 if self.verbose >= 2:
269 print("get_gerrit_lines: found %d lines" % len(lines))
272 def get_gerrits(self, project, changeid=None, limit=1, msg=None):
274 Get a list of gerrits from gerrit query request.
276 Gerrit returns queries in order of lastUpdated so resort based on merge time.
277 Also because gerrit returns them in lastUpdated order, it means all gerrits
278 merged after the one we are using will be returned, so the query limit needs to be
279 high enough to capture those extra merges plus the limit requested.
280 TODO: possibly add the before query to set a start time for the query around the change
282 :param str project: The project to search
283 :param str or None changeid: A Change-Id to search
284 :param int limit: The number of items to return
285 :param str msg: A commit-msg to search
286 :return str: A gerrit query
288 query = self.make_gerrit_query(project, changeid, limit, msg)
289 changes = self.gerrit_request(query)
290 lines = self.extract_lines_from_json(changes)
293 gerrits.append(self.parse_gerrit(line))
295 from operator import itemgetter
296 sorted_gerrits = sorted(gerrits, key=itemgetter('grantedOn'), reverse=True)
297 return sorted_gerrits