2 This module contains functions to manipulate gerrit queries.
14 # TODO: Haven't tested python 3
19 urlencode = urllib.urlencode
20 urljoin = urlparse.urljoin
21 urlparse = urlparse.urlparse
27 urlencode = urllib.parse.urlencode
28 urljoin = urllib.parse.urljoin
29 urlparse = urllib.parse.urlparse
33 logger = logging.getLogger("changes.gerritquery")
36 class GitReviewException(Exception):
40 class CommandFailed(GitReviewException):
41 """Command Failure Analysis"""
43 def __init__(self, *args):
44 Exception.__init__(self, *args)
45 (self.rc, self.output, self.argv, self.envp) = args
46 self.quickmsg = dict([
47 ("argv", " ".join(self.argv)),
49 ("output", self.output)])
52 return self.__doc__ + """
53 The following command failed with exit code %(rc)d
55 -----------------------
57 -----------------------""" % self.quickmsg
61 REMOTE_URL = 'ssh://git.opendaylight.org:29418'
65 remote_url = REMOTE_URL
67 query_limit = QUERY_LIMIT
69 def __init__(self, remote_url, branch, query_limit, verbose):
70 self.remote_url = remote_url
72 self.query_limit = query_limit
73 self.verbose = verbose
76 def print_safe_encoding(string):
78 if type(string) == unicode:
80 if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
81 encoding = sys.stdout.encoding
82 return string.encode(encoding or 'utf-8', 'replace')
88 def run_command_status(self, *argv, **kwargs):
89 logger.debug("%s Running: %s", datetime.datetime.now(), " ".join(argv))
91 # for python2 compatibility with shlex
92 if sys.version_info < (3,) and isinstance(argv[0], unicode):
93 argv = shlex.split(argv[0].encode('utf-8'))
95 argv = shlex.split(str(argv[0]))
96 stdin = kwargs.pop('stdin', None)
97 newenv = os.environ.copy()
99 newenv['LANGUAGE'] = 'C'
100 newenv.update(kwargs)
101 p = subprocess.Popen(argv,
102 stdin=subprocess.PIPE if stdin else None,
103 stdout=subprocess.PIPE,
104 stderr=subprocess.STDOUT,
106 (out, nothing) = p.communicate(stdin)
107 out = out.decode('utf-8', 'replace')
108 return p.returncode, out.strip()
110 def run_command(self, *argv, **kwargs):
111 (rc, output) = self.run_command_status(*argv, **kwargs)
114 def run_command_exc(self, klazz, *argv, **env):
116 Run command *argv, on failure raise klazz
118 klazz should be derived from CommandFailed
120 (rc, output) = self.run_command_status(*argv, **env)
122 raise klazz(rc, output, argv, env)
125 def parse_gerrit_ssh_params_from_git_url(self):
127 Parse a given Git "URL" into Gerrit parameters. Git "URLs" are either
128 real URLs or SCP-style addresses.
131 # The exact code for this in Git itself is a bit obtuse, so just do
132 # something sensible and pythonic here instead of copying the exact
135 # Handle real(ish) URLs
136 if "://" in self.remote_url:
137 parsed_url = urlparse(self.remote_url)
138 path = parsed_url.path
140 hostname = parsed_url.netloc
142 port = parsed_url.port
144 # Workaround bug in urlparse on OSX
145 if parsed_url.scheme == "ssh" and parsed_url.path[:2] == "//":
146 hostname = parsed_url.path[2:].split("/")[0]
149 (username, hostname) = hostname.split("@")
151 (hostname, port) = hostname.split(":")
156 # Handle SCP-style addresses
160 (hostname, path) = self.remote_url.split(":", 1)
162 (username, hostname) = hostname.split("@", 1)
164 # Strip leading slash and trailing .git from the path to form the project
166 project_name = re.sub(r"^/|(\.git$)", "", path)
168 return hostname, username, port, project_name
170 def gerrit_request(self, request):
172 Send a gerrit request and receive a response.
174 :param str request: A gerrit query
175 :return unicode: The JSON response
177 (hostname, username, port, project_name) = \
178 self.parse_gerrit_ssh_params_from_git_url()
180 port_data = "p%s" % port if port is not None else ""
184 userhost = "%s@%s" % (username, hostname)
186 logger.debug("gerrit request %s %s" % (self.remote_url, request))
187 output = self.run_command_exc(CommandFailed, "ssh", "-x" + port_data, userhost, request)
188 if logger.isEnabledFor(logging.DEBUG):
189 logger.debug("%s", self.print_safe_encoding(output))
192 def make_gerrit_query(self, project, changeid=None, limit=1, msg=None, status=None, comments=False, commitid=None):
194 Make a gerrit query by combining the given options.
196 :param str project: The project to search
197 :param str changeid: A Change-Id to search
198 :param int limit: The number of items to return
199 :param str msg or None: A commit-msg to search
200 :param str status or None: The gerrit status, i.e. merged
201 :param bool comments: If true include comments
202 :param commitid: A commit hash to search
203 :return str: A gerrit query
206 if project == "odlparent" or project == "yangtools":
207 query = "gerrit query --format=json limit:%d " \
211 query = "gerrit query --format=json limit:%d " \
212 "project:%s branch:%s" \
213 % (limit, project, self.branch)
215 query += " change:%s" % changeid
217 query += " message:{%s}" % msg
219 query += " commit:%s" % commitid
221 query += " status:%s --all-approvals" % status
223 query += " --comments"
226 def parse_gerrit(self, line, parse_exc=Exception):
228 Parse a single gerrit line and copy certain fields to a dictionary.
230 The merge time is found by looking for the Patch Set->Approval with
231 a SUBM type. Then use the grantedOn value.
233 :param str line: A single line from a previous gerrit query
234 :param parse_exc: The exception to except
235 :return dict: Pairs of gerrit items and their values
239 if line and line[0] == "{":
241 data = json.loads(line)
242 parsed['id'] = data['id']
243 parsed['number'] = data['number']
244 parsed['subject'] = data['subject']
245 parsed['url'] = data['url']
246 parsed['lastUpdated'] = data['lastUpdated']
247 parsed['grantedOn'] = 0
248 if "patchSets" in data:
249 patch_sets = data['patchSets']
250 for patch_set in reversed(patch_sets):
251 if "approvals" in patch_set:
252 approvals = patch_set['approvals']
253 for approval in approvals:
254 if 'type' in approval and approval['type'] == 'SUBM':
255 parsed['grantedOn'] = approval['grantedOn']
257 if parsed['grantedOn'] != 0:
259 if "comments" in data:
260 comments = data['comments']
261 for comment in reversed(comments):
262 if "message" in comment and "timestamp" in comment:
263 message = comment['message']
264 timestamp = comment['timestamp']
265 if "Build Started" in message and "patch-test" in message:
266 parsed['grantedOn'] = timestamp
269 logger.warn("Failed to decode JSON: %s", traceback.format_exc())
270 if logger.isEnabledFor(logging.DEBUG):
271 logger.warn(self.print_safe_encoding(line))
272 except Exception as err:
273 logger.warn("Exception: %s", traceback.format_exc())
277 def extract_lines_from_json(self, changes):
279 Extract a list of lines from the JSON gerrit query response.
283 :param unicode changes: The full JSON gerrit query response
284 :return list: Lines of the JSON
288 for i, line in enumerate(changes.split("\n")):
289 if line.find('"grantedOn":') != -1:
292 logger.debug("skipping: {}".format(line))
294 logger.debug("get_gerrit_lines: found {} lines, skipped: {}".format(len(lines), skipped))
297 def get_gerrits(self, project, changeid=None, limit=1, msg=None, status=None, comments=False, commitid=None):
299 Get a list of gerrits from gerrit query request.
301 Gerrit returns queries in order of lastUpdated so resort based on merge time.
302 Also because gerrit returns them in lastUpdated order, it means all gerrits
303 merged after the one we are using will be returned, so the query limit needs to be
304 high enough to capture those extra merges plus the limit requested.
305 TODO: possibly add the before query to set a start time for the query around the change
307 :param str project: The project to search
308 :param str or None changeid: A Change-Id to search
309 :param int limit: The number of items to return
310 :param str or None msg: A commit-msg to search
311 :param str or None status: The gerrit status, i.e. merged
312 :param bool comments: If true include comments
313 :param commitid: A commit hash to search
314 :return str: List of gerrits sorted by merge time
316 logger.debug("get_gerrits: project: %s, changeid: %s, limit: %d, msg: %s, status: %s, comments: %s, " +
318 project, changeid, limit, msg, status, comments, commitid)
319 query = self.make_gerrit_query(project, changeid, limit, msg, status, comments, commitid)
320 changes = self.gerrit_request(query)
321 lines = self.extract_lines_from_json(changes)
325 gerrits.append(self.parse_gerrit(line))
327 from operator import itemgetter
329 logger.warn("No gerrits were found for %s", project)
332 sorted_gerrits = sorted(gerrits, key=itemgetter('grantedOn'), reverse=True)
333 except KeyError as e:
334 logger.warn("KeyError exception in %s, %s", project, str(e))
335 return sorted_gerrits