#!/usr/bin/env python
import argparse
import gerritquery
+import logging
import os
import re
import sys
+import time
+import urllib3
import zipfile
"""
# This file started as an exact copy of git-review so including it"s copyright
COPYRIGHT = """\
-Copyright (C) 2011-2012 OpenStack LLC.
+Copyright (C) 2011-2017 OpenStack LLC.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
limitations under the License."""
-class Changes:
- VERBOSE = 0
- PROJECT_NAMES = ["genius", "mdsal", "netvirt", "neutron", "openflowjava", "openflowplugin", "ovsdb", "yangtools"]
+logger = logging.getLogger("changes")
+logger.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(levelname).4s - %(name)s - %(lineno)04d - %(message)s')
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+fh = logging.FileHandler("/tmp/changes.txt", "w")
+fh.setLevel(logging.DEBUG)
+fh.setFormatter(formatter)
+logger.addHandler(fh)
+
+
+class ChangeId(object):
+ def __init__(self, changeid, merged):
+ self.changeid = changeid
+ self.merged = merged
+
+
+class Changes(object):
+ # NETVIRT_PROJECTS, as taken from autorelease dependency info [0]
+ # TODO: it would be nice to fetch the dependency info on the fly in case it changes down the road
+ # [0] https://logs.opendaylight.org/releng/jenkins092/autorelease-release-carbon/127/archives/dependencies.log.gz
+ NETVIRT_PROJECTS = ["netvirt", "controller", "dlux", "dluxapps", "genius", "infrautils", "mdsal", "netconf",
+ "neutron", "odlparent", "openflowplugin", "ovsdb", "sfc", "yangtools"]
+ PROJECT_NAMES = NETVIRT_PROJECTS
+ VERBOSE = logging.INFO
DISTRO_PATH = "/tmp/distribution-karaf"
- REMOTE_URL = "ssh://git.opendaylight.org:29418"
+ DISTRO_URL = None
+ REMOTE_URL = gerritquery.GerritQuery.REMOTE_URL
BRANCH = "master"
LIMIT = 10
QUERY_LIMIT = 50
gerritquery = None
distro_path = DISTRO_PATH
+ distro_url = DISTRO_URL
project_names = PROJECT_NAMES
branch = BRANCH
limit = LIMIT
remote_url = REMOTE_URL
verbose = VERBOSE
projects = {}
+ regex_changeid = None
+ regex_shortmsg = None
+ regex_longmsg = None
def __init__(self, branch=BRANCH, distro_path=DISTRO_PATH,
limit=LIMIT, qlimit=QUERY_LIMIT,
self.project_names = project_names
self.remote_url = remote_url
self.verbose = verbose
- self.set_projects(project_names)
+ self.projects = {}
+ self.set_log_level(verbose)
+ self.regex_changeid = re.compile(r'(Change-Id.*: \bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b)')
+ # self.regex_shortmsg = re.compile(r'"([^"]*)"|(git.commit.message.short=(.*))')
+ self.regex_shortmsg1 = re.compile(r'(git.commit.message.short=.*"([^"]*)")')
+ self.regex_shortmsg2 = re.compile(r'(git.commit.message.short=(.*))')
+ self.regex_longmsg = re.compile(r'git.commit.message.full=(.*)')
+ self.regex_commitid = re.compile(r'(git.commit.id=(.*))')
@staticmethod
- def pretty_print_gerrits(project, gerrits):
- print("")
+ def set_log_level(level):
+ ch.setLevel(level)
+
+ def epoch_to_utc(self, epoch):
+ utc = time.gmtime(epoch)
+
+ return time.strftime("%Y-%m-%d %H:%M:%S", utc)
+
+ def pretty_print_gerrits(self, project, gerrits):
if project:
print("%s" % project)
- print("i grantedOn lastUpdatd chang subject")
- print("-- ---------- ---------- ----- -----------------------------------------")
+ print("i grantedOn lastUpdatd chang subject")
+ print("-- ------------------- ------------------- ----- -----------------------------------------")
+ if gerrits is None:
+ print("gerrit is under review")
+ return
for i, gerrit in enumerate(gerrits):
- print("%02d %d %d %s %s" % (i, gerrit["grantedOn"], gerrit["lastUpdated"],
- gerrit["number"], gerrit["subject"]))
-
- def pretty_print_includes(self, includes):
- for project, gerrits in includes.items():
- self.pretty_print_gerrits(project, gerrits)
+ if isinstance(gerrit, dict):
+ print("%02d %19s %19s %5s %s"
+ % (i,
+ self.epoch_to_utc(gerrit["grantedOn"]) if "grantedOn" in gerrit else 0,
+ self.epoch_to_utc(gerrit["lastUpdated"]) if "lastUpdated" in gerrit else 0,
+ gerrit["number"] if "number" in gerrit else "00000",
+ gerrit["subject"].encode('ascii', 'replace') if "subject" in gerrit else "none"))
def pretty_print_projects(self, projects):
- for project_name, values in projects.items():
- self.pretty_print_gerrits(project_name, values["includes"])
+ print("========================================")
+ print("distchanges")
+ print("========================================")
+ if isinstance(projects, dict):
+ for project_name, values in sorted(projects.items()):
+ if "includes" in values:
+ self.pretty_print_gerrits(project_name, values["includes"])
def set_projects(self, project_names=PROJECT_NAMES):
for project in project_names:
self.projects[project] = {"commit": [], "includes": []}
- def get_includes(self, project, changeid=None, msg=None):
+ def download_distro(self):
+ """
+ Download the distribution from self.distro_url and extract it to self.distro_path
+ """
+ logger.info("attempting to download distribution from %s and extract to %s", self.distro_url, self.distro_path)
+
+ tmp_distro_zip = '/tmp/distro.zip'
+ tmp_unzipped_location = '/tmp/distro_unzipped'
+ downloader = urllib3.PoolManager(cert_reqs='CERT_NONE')
+
+ # disabling warnings to prevent scaring the user with InsecureRequestWarning
+ urllib3.disable_warnings()
+
+ downloaded_distro = downloader.request('GET', self.distro_url)
+ with open(tmp_distro_zip, 'wb') as f:
+ f.write(downloaded_distro.data)
+
+ downloaded_distro.release_conn()
+
+ # after the .zip is extracted we want to rename it to be the distro_path which may have
+ # been given by the user
+ distro_zip = zipfile.ZipFile(tmp_distro_zip, 'r')
+ distro_zip.extractall(tmp_unzipped_location)
+ unzipped_distro_folder = os.listdir(tmp_unzipped_location)
+
+ # if the distro_path already exists, we wont overwrite it and just continue hoping what's
+ # there is relevant (and maybe already put there by this tool earlier)
+ try:
+ os.rename(tmp_unzipped_location + "/" + unzipped_distro_folder[0], self.distro_path)
+ except OSError as e:
+ logger.warn(e)
+ logger.warn("Unable to move extracted files from %s to %s. Using whatever bits are already there",
+ tmp_unzipped_location, self.distro_path)
+
+ def get_includes(self, project, changeid=None, msg=None, merged=True):
"""
Get the gerrits that would be included before the change merge time.
:param str project: The project to search
- :param str changeid: The Change-Id of the gerrit to use for the merge time
- :param str msg: The commit message of the gerrit to use for the merge time
+ :param str or None changeid: The Change-Id of the gerrit to use for the merge time
+ :param str or None msg: The commit message of the gerrit to use for the merge time
+ :param bool merged: The requested gerrit was merged
:return list: includes[0] is the gerrit requested, [1 to limit] are the gerrits found.
"""
- includes = self.gerritquery.get_gerrits(project, changeid, 1, msg)
+ if merged:
+ includes = self.gerritquery.get_gerrits(project, changeid, 1, msg, status="merged")
+ else:
+ includes = self.gerritquery.get_gerrits(project, changeid, 1, None, None, True)
if not includes:
- print("Review %s in %s:%s was not found" % (changeid, project, self.gerritquery.branch))
+ logger.info("Review %s in %s:%s was not found", changeid, project, self.gerritquery.branch)
return None
- gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg)
+ gerrits = self.gerritquery.get_gerrits(project, changeid=None, limit=self.qlimit, msg=msg, status="merged")
for gerrit in gerrits:
# don"t include the same change in the list
if gerrit["id"] == changeid:
break
if len(includes) != self.limit + 1:
- print("%s query limit was not large enough to capture %d gerrits" % (project, self.limit))
+ logger.info("%s query limit was not large enough to capture %d gerrits", project, self.limit)
return includes
zf = zipfile.ZipFile(fullpath, "r")
try:
pfile = zf.open("META-INF/git.properties")
- return pfile.read()
+ return str(pfile.read())
except KeyError:
pass
return None
- I01234567
- no Change-Id at all. There is a commit message and commit hash.
In this example the commit hash cannot be found because it was a merge
- so you must use the message. Note spaces need to be replaced with +"s
+ so you must use the message. Note spaces need to be replaced with 's.
+ - a patch that has not been merged. For these we look at the gerrit comment
+ for when the patch-test job starts.
:param str project: The project to search
:param str pfile: String containing the content of the git.properties file
- :return str: The Change-Id or None if not found
+ :return ChangeId: The Change-Id with a valid Change-Id or None if not found
"""
+ logger.info("trying Change-Id from git.properties in %s", project)
# match a 40 or 8 char Change-Id hash. both start with I
- regex = re.compile(r'\bI([a-f0-9]{40})\b|\bI([a-f0-9]{8})\b')
- changeid = regex.search(pfile)
- if changeid:
- return changeid.group()
+ changeid = self.regex_changeid.search(pfile)
+ if changeid and changeid.group(2):
+ logger.info("trying Change-Id from git.properties as merged in %s: %s", project, changeid.group(2))
+
+ gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status="merged")
+ if gerrits:
+ logger.info("found Change-Id from git.properties as merged in %s", project)
+ return ChangeId(changeid.group(2), True)
+
+ # Maybe this is a patch that has not merged yet
+ logger.info("did not find Change-Id from git.properties as merged in %s, trying as unmerged: %s",
+ project, changeid.group(2))
+
+ gerrits = self.gerritquery.get_gerrits(project, changeid.group(2), 1, None, status=None, comments=True)
+ if gerrits:
+ logger.info("found Change-Id from git.properties as unmerged in %s", project)
+ return ChangeId(gerrits[0]["id"], False)
+
+ logger.info("did not find Change-Id from git.properties in %s, trying commitid", project)
+
+ # match a git commit id
+ commitid = self.regex_commitid.search(pfile)
+ if commitid and commitid.group(2):
+ logger.info("trying commitid from git.properties in %s: %s", project, commitid.group(2))
+
+ gerrits = self.gerritquery.get_gerrits(project, commitid=commitid.group(2))
+ if gerrits:
+ logger.info("found Change-Id from git.properties as unmerged in %s", project)
+ return ChangeId(gerrits[0]["id"], True)
- # Didn"t find a Change-Id so try to get a commit message
+ logger.info("did not find Change-Id from commitid from git.properties in %s, trying short commit message1",
+ project)
+
+ # Didn't find a Change-Id so try to get a commit message
+ # match on "blah" but only keep the blah
+ msg = self.regex_shortmsg1.search(pfile)
+ if msg and msg.group(2):
+ # logger.info("msg.groups 0: %s, 1: %s, 2: %s", msg.group(), msg.group(1), msg.group(2))
+ logger.info("trying with short commit-msg 1 from git.properties in %s: %s", project, msg.group(2))
+
+ gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
+ if gerrits:
+ logger.info("found Change-Id from git.properties short commit-msg 1 in %s", project)
+ return ChangeId(gerrits[0]["id"], True)
+
+ msg_no_spaces = msg.group(2).replace(" ", "+")
+ logger.info("did not find Change-Id in %s, trying with commit-msg 1 (no spaces): %s",
+ project, msg_no_spaces)
+
+ gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
+ if gerrits:
+ logger.info("found Change-Id from git.properties short commit-msg 1 (no spaces) in %s", project)
+ return ChangeId(gerrits[0]["id"], True)
+
+ logger.info("did not find Change-Id from short commit message1 from git.properties in %s", project)
+
+ # Didn't find a Change-Id so try to get a commit message
# match on "blah" but only keep the blah
- regex_msg = re.compile(r'"([^"]*)"|^git.commit.message.short=(.*)$')
- msg = regex_msg.search(pfile)
- print("did not find Change-Id in %s, trying with commit-msg: %s" % (project, msg.group()))
+ msg = self.regex_shortmsg2.search(pfile)
+ if msg and msg.group(2):
+ logger.info("trying with short commit-msg 2 from git.properties in %s: %s", project, msg.group(2))
+
+ gerrits = self.gerritquery.get_gerrits(project, msg=msg.group(2))
+ if gerrits:
+ logger.info("found Change-Id from git.properties short commit-msg 2 in %s", project)
+ return ChangeId(gerrits[0]["id"], True)
+
+ msg_no_spaces = msg.group(2).replace(" ", "+")
+ logger.info("did not find Change-Id in %s, trying with commit-msg 2 (no spaces): %s",
+ project, msg_no_spaces)
+
+ gerrits = self.gerritquery.get_gerrits(project, msg=msg_no_spaces)
+ if gerrits:
+ logger.info("found Change-Id from git.properties short commit-msg 2 (no spaces) in %s", project)
+ return ChangeId(gerrits[0]["id"], True)
+
+ logger.info("did not find Change-Id from short commit message2 from git.properties in %s", project)
+
+ # Maybe one of the monster 'merge the world' gerrits
+ msg = self.regex_longmsg.search(pfile)
+ first_msg = None
if msg:
- # TODO: add new query using this msg
- gerrits = self.gerritquery.get_gerrits(project, None, 1, msg.group())
+ lines = str(msg.group()).split("\\n")
+ cli = next((i for i, line in enumerate(lines[:-1]) if '* changes\\:' in line), None)
+ first_msg = lines[cli + 1] if cli else None
+ if first_msg:
+ logger.info("did not find Change-Id or short commit-msg in %s, trying with merge commit-msg: %s",
+ project, first_msg)
+ gerrits = self.gerritquery.get_gerrits(project, None, 1, first_msg)
if gerrits:
- return gerrits[0]["id"]
- return None
+ logger.info("found Change-Id from git.properties merge commit-msg in %s", project)
+ return ChangeId(gerrits[0]["id"], True)
+
+ logger.warn("did not find Change-Id for %s" % project)
+
+ return ChangeId(None, False)
def find_distro_changeid(self, project):
"""
the distribution and parsing it's git.properties.
:param str project: The project to search
- :return str: The Change-Id or None if not found
+ :return ChangeId: The Change-Id with a valid Change-Id or None if not found
"""
project_dir = os.path.join(self.distro_path, "system", "org", "opendaylight", project)
pfile = None
pfile = self.extract_gitproperties_file(fullpath)
if pfile:
changeid = self.get_changeid_from_properties(project, pfile)
- if changeid:
+ if changeid.changeid:
return changeid
else:
- print("Could not find %s Change-Id in git.properties" % project)
+ logger.warn("Could not find %s Change-Id in git.properties", project)
break # all jars will have the same git.properties
if pfile is not None:
break # all jars will have the same git.properties
- return None
+ if pfile is None:
+ logger.warn("Could not find a git.properties file for %s", project)
+ return ChangeId(None, False)
def init(self):
self.gerritquery = gerritquery.GerritQuery(self.remote_url, self.branch, self.qlimit, self.verbose)
print("remote_url: %s" % self.remote_url)
print("distro_path: %s" % self.distro_path)
print("projects: %s" % (", ".join(map(str, self.projects))))
+ print("gerrit 00 is the most recent patch from which the project was built followed by the next most"
+ " recently merged patches up to %s." % self.limit)
def run_cmd(self):
"""
self.init()
self.print_options()
- for project in self.projects:
+
+ if self.distro_url is not None:
+ self.download_distro()
+
+ for project in sorted(self.projects):
+ logger.info("Processing %s", project)
changeid = self.find_distro_changeid(project)
- if changeid:
- self.projects[project]["includes"] = self.get_includes(project, changeid)
+ if changeid.changeid:
+ self.projects[project]['commit'] = changeid.changeid
+ self.projects[project]["includes"] =\
+ self.get_includes(project, changeid.changeid, msg=None, merged=changeid.merged)
return self.projects
def main(self):
help="git branch for patch under test")
parser.add_argument("-d", "--distro-path", dest="distro_path", default=self.DISTRO_PATH,
help="path to the expanded distribution, i.e. " + self.DISTRO_PATH)
+ parser.add_argument("-u", "--distro-url", dest="distro_url", default=self.DISTRO_URL,
+ help="optional url to download a distribution " + str(self.DISTRO_URL))
parser.add_argument("-l", "--limit", dest="limit", type=int, default=self.LIMIT,
help="number of gerrits to return")
parser.add_argument("-p", "--projects", dest="projects", default=self.PROJECT_NAMES,
help="number of gerrits to search")
parser.add_argument("-r", "--remote", dest="remote_url", default=self.REMOTE_URL,
help="git remote url to use for gerrit")
- parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=0,
+ parser.add_argument("-v", "--verbose", dest="verbose", action="count", default=self.VERBOSE,
help="Output more information about what's going on")
-
parser.add_argument("--license", dest="license", action="store_true",
help="Print the license and exit")
parser.add_argument("-V", "--version", action="version",
self.branch = options.branch
self.distro_path = options.distro_path
+ self.distro_url = options.distro_url
self.limit = options.limit
- self.project_names = options.projects
self.qlimit = options.qlimit
self.remote_url = options.remote_url
self.verbose = options.verbose
+ if options.projects != self.PROJECT_NAMES:
+ self.project_names = options.projects.split(',')
# TODO: add check to verify that the remote can be reached,
# though the first gerrit query will fail anyways
u = unicode(e)
except NameError:
# Python 3, we"re home free.
- print(e)
+ logger.warn(e)
else:
- print(u.encode("utf-8"))
+ logger.warn(u.encode("utf-8"))
raise
sys.exit(getattr(e, "EXIT_CODE", -1))
Code Review / integration / test.git / blobdiff