#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2016 Daniel Farrell and Others. All rights reserved.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License v1.0 which accompanies this distribution,
+# and is available at http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
"""Build OpenDaylight's RPMs using YAML build configs and Jinja2 templates."""
-import os
-import sys
import argparse
-import shutil
-import subprocess
import datetime
+import os
import re
-
+import shutil
from string import Template
+import subprocess
+import sys
from urllib2 import urlopen
-import requests
-from requests.exceptions import HTTPError
+
+import cache.cache as cache
+import specs.build_specs as build_specs
try:
- import yaml
from bs4 import BeautifulSoup
+ import requests
+ from requests.exceptions import HTTPError
+ import yaml
except ImportError:
sys.stderr.write("We recommend using our included Vagrant env.\n")
sys.stderr.write("Else, do `pip install -r requirements.txt` in a venv.\n")
raise
-import cache.cache as cache
-import specs.build_specs as build_specs
# Common paths used in this script
# This file is assumed to be in the root of the RPM build logic's dir structure
"$version_patch-$rpm_release.el7.src.rpm")
+def extract_version(url):
+ """Determine ODL version information from the ODL tarball build URL
+
+ :arg str url: URL of the ODL tarball build for building RPMs
+
+ """
+ if "autorelease" in url:
+ # Autorelease URL does not include a date and hence date extraction
+ # logic is needed for RPM versioning.
+ # Docs:
+ # https://wiki.opendaylight.org/view/Integration/Packaging/Versioning
+ # Substitute the part of the build URL not required with empty string
+ date_url = re.sub('distribution-karaf-.*\.tar\.gz$', '', url)
+ # Set date_url as an environment variable for it to be used in
+ # a subprocess
+ os.environ["date_url"] = date_url
+ # Extract ODL artifact's date by scraping data from the build URL
+ odl_date = subprocess.Popen(
+ "curl -s $date_url | grep tar.gz -A1 | tail -n1 |"
+ "sed \"s/<td>//g\" | sed \"s/\\n//g\" | awk '{print $3,$2,$6}' ",
+ shell=True, stdout=subprocess.PIPE,
+ stdin=subprocess.PIPE).stdout.read().rstrip().strip("</td>")
+ date = datetime.datetime.strptime(odl_date, "%d %b %Y").strftime(
+ '%Y%m%d')
+ # Search the ODL autorelease build URL to match the Build ID that
+ # follows "autorelease-". eg:
+ # https://nexus.opendaylight.org/content/repositories/autorelease-1533/
+ # org/opendaylight/integration/distribution-karaf/0.4.4-Beryllium-SR4/
+ # build_id = 1533
+ build_id = re.search(r'\/(autorelease)-([0-9]+)\/', url).group(2)
+ rpm_release = "0.1." + date + "rel" + build_id
+ elif "snapshot" in url:
+ # Search the ODL snapshot build URL to match the date and the Build ID
+ # that are between "distribution-karaf" and ".tar.gz".
+ # eg: https://nexus.opendaylight.org/content/repositories/
+ # opendaylight.snapshot/org/opendaylight/integration/
+ # distribution-karaf/0.6.0-SNAPSHOT/
+ # distribution-karaf-0.6.0-20161201.031047-2242.tar.gz
+ # build_id = 2242
+ # date = 20161201
+ odl_rpm = re.search(
+ r'\/(distribution-karaf)-'
+ r'([0-9]\.[0-9]\.[0-9])-([0-9]+)\.([0-9]+)-([0-9]+)\.(tar\.gz)',
+ url)
+ rpm_release = "0.1." + odl_rpm.group(3) + "snap" + odl_rpm.group(5)
+ elif "public" or "opendaylight.release" in url:
+ rpm_release = "1"
+ else:
+ raise ValueError("Unrecognized URL {}".format(url))
+
+ version = {}
+ # Search the ODL build URL to match 0.major.minor-codename-SR and extract
+ # version information. eg: release:
+ # https://nexus.opendaylight.org/content/repositories/public/org/
+ # opendaylight/integration/distribution-karaf/0.3.3-Lithium-SR3/
+ # distribution-karaf-0.3.3-Lithium-SR3.tar.gz
+ # match: 0.3.3-Lithium-SR3
+ odl_version = re.search(r'\/(\d)\.(\d)\.(\d).(.*)\/', url)
+ version["version_major"] = odl_version.group(2)
+ version["version_minor"] = odl_version.group(3)
+ version["version_patch"] = "0"
+ version["rpm_release"] = rpm_release
+ version["codename"] = odl_version.group(4)
+ return version
+
+
def build_rpm(build):
- """Build the RPMs described by the given build description.
+ """Build the RPMs described by the given build description
:param build: Description of an RPM build, typically from build_vars.yaml
:type build: dict
# else, find the latest sub-directory
sub_dir = ''
snapshot_dir = ''
- if build['version_minor']:
+ try:
sub_dir = '0.' + build['version_major'] + '.' + \
build['version_minor'] + '-SNAPSHOT/'
snapshot_dir = parent_dir + sub_dir
- else:
+ except KeyError:
subdir_url = urlopen(parent_dir)
content = subdir_url.read().decode('utf-8')
all_dirs = BeautifulSoup(content, 'html.parser')
# Get download_url
build['download_url'] = snapshot_url
- # Get changelog_date from the snapshot URL
- # eg: 'distribution-karaf-0.5.2-20161202.230609-363.tar.gz'
- # '\d{8}' searches for the date in the url
- extract_date = re.search(r'\d{8}', snapshot_url)
- extract_date = extract_date.group(0)
- year = int(extract_date[:4])
- month = int(extract_date[4:6])
- date = int(extract_date[6:])
-
- # %a: Abbreviated weekday name
- # %b: Abbreviated month name
- # %d: Zero padded decimal number
- # %Y: Year
- # `changelog_date` is in the format: 'Sat Dec 10 2016'
- # Docs:
- # https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
- build['changelog_date'] = datetime.date(year,
- month,
- date).strftime("%a %b %d %Y")
-
- # Assign codename
- build['codename'] = "SNAPSHOT"
+ # Call `extract_version` function to get version information
+ # except Major and Minor version which are already present
+ version = extract_version(build['download_url'])
+ build['version_patch'] = version['version_patch']
+ build['rpm_release'] = version['rpm_release']
+ build['codename'] = version['codename']
urlpath.close()
build_rpm(build)
+def get_sysd_commit():
+ """Get latest Int/Pack repo commit hash"""
+
+ int_pack_repo = "https://github.com/opendaylight/integration-packaging.git"
+ # Get the commit hash at the tip of the master branch
+ args_git = ['git', 'ls-remote', int_pack_repo, "HEAD"]
+ args_awk = ['awk', '{print $1}']
+ references = subprocess.Popen(args_git, stdout=subprocess.PIPE,
+ shell=False)
+ sysd_commit = subprocess.check_output(args_awk, stdin=references.stdout,
+ shell=False).strip()
+
+ return sysd_commit
+
+
# When run as a script, accept a set of builds and execute them
if __name__ == "__main__":
- # Load RPM build variables from a YAML config file
- build_vars_path = os.path.join(project_root, "build_vars.yaml")
- with open(build_vars_path) as rpm_var_file:
- build_vars = yaml.load(rpm_var_file)
-
# Accept the version(s) of the build(s) to perform as args
# TODO: More docs on ArgParser and argument
parser = argparse.ArgumentParser(conflict_handler='resolve')
- existing_build_group = parser.add_argument_group("Existing build")
- existing_build_group.add_argument(
- "-v", "--version", action="append", metavar="major minor patch rpm",
- nargs="*", help="RPM version(s) to build"
- )
+
new_build_group = parser.add_argument_group("New build")
new_build_group.add_argument(
- "--major", help="Major (element) version to build")
- new_build_group.add_argument("--minor", help="Minor (SR) version to build")
- new_build_group.add_argument("--patch", help="Patch version to build")
- new_build_group.add_argument("--rpm", help="RPM version to build")
+ "--download_url", help="Tarball to repackage into RPM")
new_build_group.add_argument(
"--sysd_commit", help="Version of ODL unitfile to package")
- new_build_group.add_argument("--codename", help="Codename for ODL version")
- new_build_group.add_argument(
- "--download_url", help="Tarball to repackage into RPM")
new_build_group.add_argument(
"--changelog_date", help="Date this RPM was defined")
new_build_group.add_argument(
latest_snap_group = parser.add_argument_group("Latest snapshot build")
latest_snap_group.add_argument("--build-latest-snap", action='store_true',
help="Build RPM from the latest snpashot")
- latest_snap_group.add_argument("--major", required='true',
- help="Stable branch from which "
+ latest_snap_group.add_argument("--major", help="Stable branch from which "
"to build the snapshot")
latest_snap_group.add_argument("--minor", help="Minor version of the "
"stable branch to build the snapshot")
- latest_snap_group.add_argument("--patch", help="Patch version to build")
- latest_snap_group.add_argument("--rpm", help="RPM version to build")
latest_snap_group.add_argument("--sysd_commit",
help="Version of ODL unitfile to package")
- latest_snap_group.add_argument("--codename",
- help="Codename for ODL snapshot")
latest_snap_group.add_argument("--changelog_name",
help="Name of person who defined RPM")
latest_snap_group.add_argument("--changelog_email",
# Parse the given args
args = parser.parse_args()
- # Build list of RPM builds to perform
- builds = []
- if args.version:
- # Build a list of requested versions as dicts of version components
- versions = []
- version_keys = ["version_major", "version_minor", "version_patch",
- "rpm_release"]
- # For each version arg, match all version components to build_vars name
- for version in args.version:
- versions.append(dict(zip(version_keys, version)))
-
- # Find every RPM build that matches any version argument
- # A passed version "matches" a build when the provided version
- # components are a subset of the version components of a build. Any
- # version components that aren't passed are simply not checked, so
- # they can't fail the match, effectively wild-carding them.
- for build in build_vars["builds"]:
- for version in versions:
- # Converts both dicts' key:value pairs to lists of tuples and
- # checks that each tuple in the version list is present in the
- # build list.
- if all(item in build.items() for item in version.items()):
- builds.append(build)
- else:
- builds.append({"version_major": args.major,
- "version_minor": args.minor,
- "version_patch": args.patch,
- "rpm_release": args.rpm,
- "sysd_commit": args.sysd_commit,
- "codename": args.codename,
- "download_url": args.download_url,
- "changelog_date": args.changelog_date,
- "changelog_name": args.changelog_name,
- "changelog_email": args.changelog_email})
+ # A dictionary containing essential build variables
+ build = {}
+ # Check if `changelog_date` has been passed as an arg
+ # The current datetime should be the default date for RPM changelog date
+ # but can still accept optional `changelog_date` param
+ # `changelog_date` is in the format: 'Sat Dec 10 2016'
+ # Docs:
+ # https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
+ if not args.changelog_date:
+ args.changelog_date = datetime.date.today().strftime("%a %b %d %Y")
+
+ # Check if `sysd_commit` has been passed as an arg
+ # Use latest Int/Pack repo commit hash as sysd_commit var
+ # unless passed by param
+ if not args.sysd_commit:
+ args.sysd_commit = get_sysd_commit()
+
+ # If download_url is given, update version info
+ if args.download_url:
+ build.update({"download_url": args.download_url})
+ version = extract_version(args.download_url)
+ build.update(version)
+
+ # Common parameters for all new and snapshot builds
+ build.update({"sysd_commit": args.sysd_commit,
+ "changelog_name": args.changelog_name,
+ "changelog_email": args.changelog_email,
+ "changelog_date": args.changelog_date,
+ })
# If the flag `--build-latest-snap` is true, extract information
- # from the snapshot URL, else directly build the RPM
- for build in builds:
- if args.build_latest_snap:
+ # from the snapshot URL using major version and minor version(optional)
+ # info, else proceed directly to build the RPM
+ if args.build_latest_snap:
+ if args.major:
+ build.update({'version_major': args.major})
+ if args.minor:
+ build.update({'version_minor': args.minor})
build_snapshot_rpm(build)
- else:
- build_rpm(build)
+ else:
+ build_rpm(build)
Code Review / integration / packaging.git / blobdiff