X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=tools%2Fdistchanges%2Fchanges.py;h=50ac8661cc7c1b2e1449a9a47e5d3487b66cc422;hb=7259a2a607f9a99344833d4bb50151ded5fbec07;hp=29825a09dabedd071e5f8cb179050ea8bd20be0f;hpb=1211d3c599cbe0635771442dc4ad088b571f9c31;p=integration%2Ftest.git diff --git a/tools/distchanges/changes.py b/tools/distchanges/changes.py index 29825a09da..50ac8661cc 100644 --- a/tools/distchanges/changes.py +++ b/tools/distchanges/changes.py @@ -3,8 +3,8 @@ import argparse import gerritquery import os import re -import shutil import sys +import time import urllib3 import zipfile @@ -39,7 +39,13 @@ See the License for the specific language governing permissions and limitations under the License.""" -class Changes: +class ChangeId(object): + def __init__(self, changeid, merged): + self.changeid = changeid + self.merged = merged + + +class Changes(object): # NETVIRT_PROJECTS, as taken from autorelease dependency info [0] # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz @@ -76,26 +82,36 @@ class Changes: self.project_names = project_names self.remote_url = remote_url self.verbose = verbose + self.projects = {} - @staticmethod - def pretty_print_gerrits(project, gerrits): + def epoch_to_utc(self, epoch): + utc = time.gmtime(epoch) + + return time.strftime("%Y-%m-%d %H:%M:%S", utc) + + def pretty_print_gerrits(self, project, gerrits): print("") if project: print("%s" % project) - print("i grantedOn lastUpdatd chang subject") - print("-- ---------- ---------- ----- -----------------------------------------") + print("i grantedOn lastUpdatd chang subject") + print("-- ------------------- ------------------- ----- -----------------------------------------") + if gerrits is None: + print("gerrit is under review") + return for i, gerrit in enumerate(gerrits): - print("%02d %d %d %s %s" % (i, gerrit["grantedOn"], gerrit["lastUpdated"], - gerrit["number"], gerrit["subject"])) - - def pretty_print_includes(self, includes): - for project, gerrits in includes.items(): - self.pretty_print_gerrits(project, gerrits) + if isinstance(gerrit, dict): + print("%02d %19s %19s %5s %s" + % (i, + self.epoch_to_utc(gerrit["grantedOn"]) if "grantedOn" in gerrit else 0, + self.epoch_to_utc(gerrit["lastUpdated"]) if "lastUpdated" in gerrit else 0, + gerrit["number"] if "number" in gerrit else "00000", + gerrit["subject"] if "subject" in gerrit else "none")) def pretty_print_projects(self, projects): - for project_name, values in projects.items(): - if values["includes"]: - self.pretty_print_gerrits(project_name, values["includes"]) + if isinstance(projects, dict): + for project_name, values in sorted(projects.items()): + if "includes" in values: + self.pretty_print_gerrits(project_name, values["includes"]) def set_projects(self, project_names=PROJECT_NAMES): for project in project_names: @@ -137,21 +153,25 @@ class Changes: print("Unable to move extracted files from %s to %s. Using whatever bits are already there" % (tmp_unzipped_location, self.distro_path)) - def get_includes(self, project, changeid=None, msg=None): + def get_includes(self, project, changeid=None, msg=None, merged=True): """ Get the gerrits that would be included before the change merge time. :param str project: The project to search - :param str changeid: The Change-Id of the gerrit to use for the merge time - :param str msg: The commit message of the gerrit to use for the merge time + :param str or None changeid: The Change-Id of the gerrit to use for the merge time + :param str or None msg: The commit message of the gerrit to use for the merge time + :param bool merged: The requested gerrit was merged :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found. """ - includes = self.gerritquery.get_gerrits(project, changeid, 1, msg) + if merged: + includes = self.gerritquery.get_gerrits(project, changeid, 1, msg, status="merged") + else: + includes = self.gerritquery.get_gerrits(project, changeid, 1, None, None, True) if not includes: print("Review %s in %s:%s was not found" % (changeid, project, self.gerritquery.branch)) return None - gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg) + gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg, status="merged") for gerrit in gerrits: # don"t include the same change in the list if gerrit["id"] == changeid: @@ -182,7 +202,7 @@ class Changes: zf = zipfile.ZipFile(fullpath, "r") try: pfile = zf.open("META-INF/git.properties") - return pfile.read() + return str(pfile.read()) except KeyError: pass return None @@ -196,31 +216,72 @@ class Changes: - I01234567 - no Change-Id at all. There is a commit message and commit hash. In this example the commit hash cannot be found because it was a merge - so you must use the message. Note spaces need to be replaced with +"s + so you must use the message. Note spaces need to be replaced with 's. + - a patch that has not been merged. For these we look at the gerrit comment + for when the patch-test job starts. :param str project: The project to search :param str pfile: String containing the content of the git.properties file - :return str: The Change-Id or None if not found + :return ChangeId: The Change-Id with a valid Change-Id or None if not found """ # match a 40 or 8 char Change-Id hash. both start with I regex = re.compile(r'\bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b') changeid = regex.search(pfile) if changeid: - return changeid.group() + if self.verbose >= 1: + print("trying Change-Id as merged in %s" % (project)) - # Didn"t find a Change-Id so try to get a commit message + gerrits = self.gerritquery.get_gerrits(project, changeid.group(), 1, None, status="merged") + if gerrits: + return ChangeId(changeid.group(), True) + + # Maybe this is a patch that has not merged yet + if self.verbose >= 1: + print("did not find Change-Id as merged in %s, trying as unmerged" % project) + + gerrits = self.gerritquery.get_gerrits(project, changeid.group(), 1, None, status=None, comments=True) + if gerrits: + return ChangeId(gerrits[0]["id"], False) + + # Didn't find a Change-Id so try to get a commit message # match on "blah" but only keep the blah regex_msg = re.compile(r'"([^"]*)"|^git.commit.message.short=(.*)$') msg = regex_msg.search(pfile) - if self.verbose >= 2: - print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group())) - if msg: - # TODO: add new query using this msg + if self.verbose >= 1: + print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group())) + gerrits = self.gerritquery.get_gerrits(project, None, 1, msg.group()) if gerrits: - return gerrits[0]["id"] - return None + return ChangeId(gerrits[0]["id"], True) + + msg_no_spaces = msg.group().replace(" ", "+") + if self.verbose >= 1: + print("did not find Change-Id in %s, trying with commit-msg (no spaces): %s" % (project, msg_no_spaces)) + + gerrits = self.gerritquery.get_gerrits(project, None, 1, msg_no_spaces) + if gerrits: + return ChangeId(gerrits[0]["id"], True) + + # Maybe one of the monster 'merge the world' gerrits + regex_msg = re.compile(r'git.commit.message.full=(.*)') + msg = regex_msg.search(pfile) + first_msg = None + if msg: + lines = str(msg.group()).split("\\n") + cli = next((i for i, line in enumerate(lines[:-1]) if '* changes\\:' in line), None) + first_msg = lines[cli+1] if cli else None + if first_msg: + if self.verbose >= 1: + print("did not find Change-Id or commit-msg in %s, trying with merge commit-msg: %s" + % (project, first_msg)) + gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg) + if gerrits: + return ChangeId(gerrits[0]["id"], True) + + print("did not find Change-Id for %s" % project) + + return ChangeId(None, False) def find_distro_changeid(self, project): """ @@ -228,7 +289,7 @@ class Changes: the distribution and parsing it's git.properties. :param str project: The project to search - :return str: The Change-Id or None if not found + :return ChangeId: The Change-Id with a valid Change-Id or None if not found """ project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project) pfile = None @@ -239,14 +300,16 @@ class Changes: pfile = self.extract_gitproperties_file(fullpath) if pfile: changeid = self.get_changeid_from_properties(project, pfile) - if changeid: + if changeid.changeid: return changeid else: print("Could not find %s Change-Id in git.properties" % project) break # all jars will have the same git.properties if pfile is not None: break # all jars will have the same git.properties - return None + if pfile is None: + print("Could not find a git.properties file for %s" % project) + return ChangeId(None, False) def init(self): self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose) @@ -278,10 +341,14 @@ class Changes: if self.distro_url is not None: self.download_distro() - for project in self.projects: + for project in sorted(self.projects): + if self.verbose >= 1: + print("Processing %s" % project) changeid = self.find_distro_changeid(project) - if changeid: - self.projects[project]["includes"] = self.get_includes(project, changeid) + if changeid.changeid: + self.projects[project]['commit'] = changeid.changeid + self.projects[project]["includes"] =\ + self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged) return self.projects def main(self):