X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=tools%2Fdistchanges%2Fchanges.py;h=d7cb99ad35079e40208c89bc9fcef72acb00fe5b;hb=5374ad49290b5c25e667858f9a9e514b6c7cb958;hp=cb64fa15e92232dcd89614abe7be96ec24f09098;hpb=58dfa96b4c05f0054496a060b339fea50d0a31a3;p=integration%2Ftest.git diff --git a/tools/distchanges/changes.py b/tools/distchanges/changes.py index cb64fa15e9..d7cb99ad35 100644 --- a/tools/distchanges/changes.py +++ b/tools/distchanges/changes.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import argparse import gerritquery +import logging import os import re -import shutil import sys +import time import urllib3 import zipfile @@ -39,14 +40,33 @@ See the License for the specific language governing permissions and limitations under the License.""" -class Changes: +logger = logging.getLogger("changes") +logger.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s') +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +ch.setFormatter(formatter) +logger.addHandler(ch) +fh = logging.FileHandler("/tmp/changes.txt", "w") +fh.setLevel(logging.DEBUG) +fh.setFormatter(formatter) +logger.addHandler(fh) + + +class ChangeId(object): + def __init__(self, changeid, merged): + self.changeid = changeid + self.merged = merged + + +class Changes(object): # NETVIRT_PROJECTS, as taken from autorelease dependency info [0] # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz - NETVIRT_PROJECTS = ["controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf", "neutron", - "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"] + NETVIRT_PROJECTS = ["netvirt", "controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf", + "neutron", "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"] PROJECT_NAMES = NETVIRT_PROJECTS - VERBOSE = 0 + VERBOSE = logging.INFO DISTRO_PATH = "/tmp/distribution-karaf" DISTRO_URL = None REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL @@ -64,6 +84,9 @@ class Changes: remote_url = REMOTE_URL verbose = VERBOSE projects = {} + regex_changeid = None + regex_shortmsg = None + regex_longmsg = None def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH, limit=LIMIT, qlimit=QUERY_LIMIT, @@ -76,26 +99,49 @@ class Changes: self.project_names = project_names self.remote_url = remote_url self.verbose = verbose - self.set_projects(project_names) + self.projects = {} + self.set_log_level(verbose) + self.regex_changeid = re.compile(r'(Change-Id.*: \bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)') + # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))') + self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")') + self.regex_shortmsg2 = re.compile(r'(git.commit.message.short=(.*))') + self.regex_longmsg = re.compile(r'git.commit.message.full=(.*)') + self.regex_commitid = re.compile(r'(git.commit.id=(.*))') @staticmethod - def pretty_print_gerrits(project, gerrits): - print("") + def set_log_level(level): + ch.setLevel(level) + + def epoch_to_utc(self, epoch): + utc = time.gmtime(epoch) + + return time.strftime("%Y-%m-%d %H:%M:%S", utc) + + def pretty_print_gerrits(self, project, gerrits): if project: print("%s" % project) - print("i grantedOn lastUpdatd chang subject") - print("-- ---------- ---------- ----- -----------------------------------------") + print("i grantedOn lastUpdatd chang subject") + print("-- ------------------- ------------------- ----- -----------------------------------------") + if gerrits is None: + print("gerrit is under review") + return for i, gerrit in enumerate(gerrits): - print("%02d %d %d %s %s" % (i, gerrit["grantedOn"], gerrit["lastUpdated"], - gerrit["number"], gerrit["subject"])) - - def pretty_print_includes(self, includes): - for project, gerrits in includes.items(): - self.pretty_print_gerrits(project, gerrits) + if isinstance(gerrit, dict): + print("%02d %19s %19s %5s %s" + % (i, + self.epoch_to_utc(gerrit["grantedOn"]) if "grantedOn" in gerrit else 0, + self.epoch_to_utc(gerrit["lastUpdated"]) if "lastUpdated" in gerrit else 0, + gerrit["number"] if "number" in gerrit else "00000", + gerrit["subject"].encode('ascii', 'replace') if "subject" in gerrit else "none")) def pretty_print_projects(self, projects): - for project_name, values in projects.items(): - self.pretty_print_gerrits(project_name, values["includes"]) + print("========================================") + print("distchanges") + print("========================================") + if isinstance(projects, dict): + for project_name, values in sorted(projects.items()): + if "includes" in values: + self.pretty_print_gerrits(project_name, values["includes"]) def set_projects(self, project_names=PROJECT_NAMES): for project in project_names: @@ -105,9 +151,7 @@ class Changes: """ Download the distribution from self.distro_url and extract it to self.distro_path """ - if self.verbose >= 2: - print("attempting to download distribution from %s and extract to %s " % - (self.distro_url, self.distro_path)) + logger.info("attempting to download distribution from %s and extract to %s", self.distro_url, self.distro_path) tmp_distro_zip = '/tmp/distro.zip' tmp_unzipped_location = '/tmp/distro_unzipped' @@ -133,25 +177,29 @@ class Changes: try: os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path) except OSError as e: - print(e) - print("Unable to move extracted files from %s to %s. Using whatever bits are already there" % - (tmp_unzipped_location, self.distro_path)) + logger.warn(e) + logger.warn("Unable to move extracted files from %s to %s. Using whatever bits are already there", + tmp_unzipped_location, self.distro_path) - def get_includes(self, project, changeid=None, msg=None): + def get_includes(self, project, changeid=None, msg=None, merged=True): """ Get the gerrits that would be included before the change merge time. :param str project: The project to search - :param str changeid: The Change-Id of the gerrit to use for the merge time - :param str msg: The commit message of the gerrit to use for the merge time + :param str or None changeid: The Change-Id of the gerrit to use for the merge time + :param str or None msg: The commit message of the gerrit to use for the merge time + :param bool merged: The requested gerrit was merged :return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found. """ - includes = self.gerritquery.get_gerrits(project, changeid, 1, msg) + if merged: + includes = self.gerritquery.get_gerrits(project, changeid, 1, msg, status="merged") + else: + includes = self.gerritquery.get_gerrits(project, changeid, 1, None, None, True) if not includes: - print("Review %s in %s:%s was not found" % (changeid, project, self.gerritquery.branch)) + logger.info("Review %s in %s:%s was not found", changeid, project, self.gerritquery.branch) return None - gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg) + gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg, status="merged") for gerrit in gerrits: # don"t include the same change in the list if gerrit["id"] == changeid: @@ -166,7 +214,7 @@ class Changes: break if len(includes) != self.limit + 1: - print("%s query limit was not large enough to capture %d gerrits" % (project, self.limit)) + logger.info("%s query limit was not large enough to capture %d gerrits", project, self.limit) return includes @@ -182,7 +230,7 @@ class Changes: zf = zipfile.ZipFile(fullpath, "r") try: pfile = zf.open("META-INF/git.properties") - return pfile.read() + return str(pfile.read()) except KeyError: pass return None @@ -196,31 +244,112 @@ class Changes: - I01234567 - no Change-Id at all. There is a commit message and commit hash. In this example the commit hash cannot be found because it was a merge - so you must use the message. Note spaces need to be replaced with +"s + so you must use the message. Note spaces need to be replaced with 's. + - a patch that has not been merged. For these we look at the gerrit comment + for when the patch-test job starts. :param str project: The project to search :param str pfile: String containing the content of the git.properties file - :return str: The Change-Id or None if not found + :return ChangeId: The Change-Id with a valid Change-Id or None if not found """ + logger.info("trying Change-Id from git.properties in %s", project) # match a 40 or 8 char Change-Id hash. both start with I - regex = re.compile(r'\bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b') - changeid = regex.search(pfile) - if changeid: - return changeid.group() + changeid = self.regex_changeid.search(pfile) + if changeid and changeid.group(2): + logger.info("trying Change-Id from git.properties as merged in %s: %s", project, changeid.group(2)) + + gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status="merged") + if gerrits: + logger.info("found Change-Id from git.properties as merged in %s", project) + return ChangeId(changeid.group(2), True) + + # Maybe this is a patch that has not merged yet + logger.info("did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s", + project, changeid.group(2)) + + gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status=None, comments=True) + if gerrits: + logger.info("found Change-Id from git.properties as unmerged in %s", project) + return ChangeId(gerrits[0]["id"], False) + + logger.info("did not find Change-Id from git.properties in %s, trying commitid", project) + + # match a git commit id + commitid = self.regex_commitid.search(pfile) + if commitid and commitid.group(2): + logger.info("trying commitid from git.properties in %s: %s", project, commitid.group(2)) + + gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2)) + if gerrits: + logger.info("found Change-Id from git.properties as unmerged in %s", project) + return ChangeId(gerrits[0]["id"], True) + + logger.info("did not find Change-Id from commitid from git.properties in %s, trying short commit message1", + project) - # Didn"t find a Change-Id so try to get a commit message + # Didn't find a Change-Id so try to get a commit message # match on "blah" but only keep the blah - regex_msg = re.compile(r'"([^"]*)"|^git.commit.message.short=(.*)$') - msg = regex_msg.search(pfile) - if self.verbose >= 2: - print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group())) + msg = self.regex_shortmsg1.search(pfile) + if msg and msg.group(2): + # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2)) + logger.info("trying with short commit-msg 1 from git.properties in %s: %s", project, msg.group(2)) + gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2)) + if gerrits: + logger.info("found Change-Id from git.properties short commit-msg 1 in %s", project) + return ChangeId(gerrits[0]["id"], True) + + msg_no_spaces = msg.group(2).replace(" ", "+") + logger.info("did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s", + project, msg_no_spaces) + + gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces) + if gerrits: + logger.info("found Change-Id from git.properties short commit-msg 1 (no spaces) in %s", project) + return ChangeId(gerrits[0]["id"], True) + + logger.info("did not find Change-Id from short commit message1 from git.properties in %s", project) + + # Didn't find a Change-Id so try to get a commit message + # match on "blah" but only keep the blah + msg = self.regex_shortmsg2.search(pfile) + if msg and msg.group(2): + logger.info("trying with short commit-msg 2 from git.properties in %s: %s", project, msg.group(2)) + + gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2)) + if gerrits: + logger.info("found Change-Id from git.properties short commit-msg 2 in %s", project) + return ChangeId(gerrits[0]["id"], True) + + msg_no_spaces = msg.group(2).replace(" ", "+") + logger.info("did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s", + project, msg_no_spaces) + + gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces) + if gerrits: + logger.info("found Change-Id from git.properties short commit-msg 2 (no spaces) in %s", project) + return ChangeId(gerrits[0]["id"], True) + + logger.info("did not find Change-Id from short commit message2 from git.properties in %s", project) + + # Maybe one of the monster 'merge the world' gerrits + msg = self.regex_longmsg.search(pfile) + first_msg = None if msg: - # TODO: add new query using this msg - gerrits = self.gerritquery.get_gerrits(project, None, 1, msg.group()) + lines = str(msg.group()).split("\\n") + cli = next((i for i, line in enumerate(lines[:-1]) if '* changes\\:' in line), None) + first_msg = lines[cli + 1] if cli else None + if first_msg: + logger.info("did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s", + project, first_msg) + gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg) if gerrits: - return gerrits[0]["id"] - return None + logger.info("found Change-Id from git.properties merge commit-msg in %s", project) + return ChangeId(gerrits[0]["id"], True) + + logger.warn("did not find Change-Id for %s" % project) + + return ChangeId(None, False) def find_distro_changeid(self, project): """ @@ -228,7 +357,7 @@ class Changes: the distribution and parsing it's git.properties. :param str project: The project to search - :return str: The Change-Id or None if not found + :return ChangeId: The Change-Id with a valid Change-Id or None if not found """ project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project) pfile = None @@ -239,14 +368,16 @@ class Changes: pfile = self.extract_gitproperties_file(fullpath) if pfile: changeid = self.get_changeid_from_properties(project, pfile) - if changeid: + if changeid.changeid: return changeid else: - print("Could not find %s Change-Id in git.properties" % project) + logger.warn("Could not find %s Change-Id in git.properties", project) break # all jars will have the same git.properties if pfile is not None: break # all jars will have the same git.properties - return None + if pfile is None: + logger.warn("Could not find a git.properties file for %s", project) + return ChangeId(None, False) def init(self): self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose) @@ -278,10 +409,13 @@ class Changes: if self.distro_url is not None: self.download_distro() - for project in self.projects: + for project in sorted(self.projects): + logger.info("Processing %s", project) changeid = self.find_distro_changeid(project) - if changeid: - self.projects[project]["includes"] = self.get_includes(project, changeid) + if changeid.changeid: + self.projects[project]['commit'] = changeid.changeid + self.projects[project]["includes"] =\ + self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged) return self.projects def main(self): @@ -319,10 +453,11 @@ class Changes: self.distro_path = options.distro_path self.distro_url = options.distro_url self.limit = options.limit - self.project_names = options.projects self.qlimit = options.qlimit self.remote_url = options.remote_url self.verbose = options.verbose + if options.projects != self.PROJECT_NAMES: + self.project_names = options.projects.split(',') # TODO: add check to verify that the remote can be reached, # though the first gerrit query will fail anyways @@ -344,9 +479,9 @@ def main(): u = unicode(e) except NameError: # Python 3, we"re home free. - print(e) + logger.warn(e) else: - print(u.encode("utf-8")) + logger.warn(u.encode("utf-8")) raise sys.exit(getattr(e, "EXIT_CODE", -1))