rpm/build.py

   1 #!/usr/bin/env python
   2 """Build OpenDaylight's RPMs using YAML build configs and Jinja2 templates."""
   3
   4 import os
   5 import sys
   6 import argparse
   7 import shutil
   8 import subprocess
   9 import datetime
  10 import re
  11
  12 from string import Template
  13 from urllib2 import urlopen
  14 import requests
  15 from requests.exceptions import HTTPError
  16
  17 try:
  18     import yaml
  19     from bs4 import BeautifulSoup
  20 except ImportError:
  21     sys.stderr.write("We recommend using our included Vagrant env.\n")
  22     sys.stderr.write("Else, do `pip install -r requirements.txt` in a venv.\n")
  23     raise
  24
  25 import cache.cache as cache
  26 import specs.build_specs as build_specs
  27
  28 # Common paths used in this script
  29 # This file is assumed to be in the root of the RPM build logic's dir structure
  30 project_root = os.path.dirname(os.path.abspath(__file__))
  31 cache_dir = os.path.join(project_root, "cache")
  32 specs_dir = os.path.join(project_root, "specs")
  33 rpmbuild_dir = os.path.join(os.path.expanduser("~"), "rpmbuild")
  34 src_in_dir = os.path.join(rpmbuild_dir, "SOURCES")
  35 spec_in_dir = os.path.join(rpmbuild_dir, "SPECS")
  36 srpm_out_dir = os.path.join(rpmbuild_dir, "SRPMS")
  37 rpm_out_dir = os.path.join(rpmbuild_dir, "RPMS", "noarch")
  38
  39 # Templates that can be specialized into common artifact names per-build
  40 odl_template = Template("opendaylight-$version_major.$version_minor."
  41                         "$version_patch-$rpm_release.tar.gz")
  42 specfile_template = Template("opendaylight-$version_major.$version_minor."
  43                              "$version_patch-$rpm_release.spec")
  44 unitfile_tb_template = Template("opendaylight-$sysd_commit.service.tar.gz")
  45 rpm_template = Template("opendaylight-$version_major.$version_minor."
  46                         "$version_patch-$rpm_release.el7.noarch.rpm")
  47 srpm_template = Template("opendaylight-$version_major.$version_minor."
  48                          "$version_patch-$rpm_release.el7.src.rpm")
  49
  50
  51 def build_rpm(build):
  52     """Build the RPMs described by the given build description.
  53
  54     :param build: Description of an RPM build, typically from build_vars.yaml
  55     :type build: dict
  56
  57     """
  58     # Specialize a series of name templates for the given build
  59     odl_tarball = odl_template.substitute(build)
  60     odl_rpm = rpm_template.substitute(build)
  61     odl_srpm = srpm_template.substitute(build)
  62     odl_specfile = specfile_template.substitute(build)
  63     unitfile_tarball = unitfile_tb_template.substitute(build)
  64
  65     # After building strings from the name templates, build their full path
  66     odl_tarball_path = os.path.join(cache_dir, odl_tarball)
  67     unitfile_tarball_path = os.path.join(cache_dir, unitfile_tarball)
  68     specfile_path = os.path.join(specs_dir, odl_specfile)
  69     spec_in_path = os.path.join(spec_in_dir, odl_specfile)
  70     rpm_out_path = os.path.join(rpm_out_dir, odl_rpm)
  71     srpm_out_path = os.path.join(srpm_out_dir, odl_srpm)
  72
  73     # Call a helper function to cache the artifacts required for each build
  74     cache.cache_build(build)
  75
  76     # Call helper script to build the required RPM .spec files
  77     build_specs.build_spec(build)
  78
  79     # Clean up old rpmbuild dir structure if it exists
  80     if os.path.isdir(rpmbuild_dir):
  81         shutil.rmtree(rpmbuild_dir)
  82
  83     # Create rpmbuild dir structure
  84     subprocess.call("rpmdev-setuptree")
  85
  86     # Move unitfile, tarball and specfile to correct rpmbuild dirs
  87     shutil.copy(odl_tarball_path, src_in_dir)
  88     shutil.copy(unitfile_tarball_path, src_in_dir)
  89     shutil.copy(specfile_path, spec_in_dir)
  90
  91     # Call rpmbuild, build both SRPMs/RPMs
  92     subprocess.call(["rpmbuild", "-ba", spec_in_path])
  93
  94     # Copy the RPMs/SRPMs from their output dir to the cache dir
  95     shutil.copy(rpm_out_path, cache_dir)
  96     shutil.copy(srpm_out_path, cache_dir)
  97
  98
  99 def build_snapshot_rpm(build):
 100     """Build latest snapshot RPMs fetching information from URL.
 101
 102     :param build: Description of an RPM build, from parent_dir URL
 103     :type build: dict
 104
 105     """
 106     parent_dir = "https://nexus.opendaylight.org/content/repositories/" \
 107                  "opendaylight.snapshot/org/opendaylight/integration/"\
 108                  "distribution-karaf/"
 109
 110     # If the minor verison is given, get the sub-directory directly
 111     # else, find the latest sub-directory
 112     sub_dir = ''
 113     snapshot_dir = ''
 114     if build['version_minor']:
 115         sub_dir = '0.' + build['version_major'] + '.' + \
 116                    build['version_minor'] + '-SNAPSHOT/'
 117         snapshot_dir = parent_dir + sub_dir
 118     else:
 119         subdir_url = urlopen(parent_dir)
 120         content = subdir_url.read().decode('utf-8')
 121         all_dirs = BeautifulSoup(content, 'html.parser')
 122
 123         # Loops through all the sub-directories present and stores the
 124         # latest sub directory as sub-directories are already sorted
 125         # in early to late order.
 126         for tag in all_dirs.find_all('a', href=True):
 127             # Checks if the sub-directory name is of the form
 128             # '0.<major_version>.<minor_version>-SNAPSHOT'.
 129             dir = re.search(r'\/(\d)\.(\d)\.(\d).(.*)\/', tag['href'])
 130             # If the major version matches the argument provided
 131             # store the minor version, else ignore.
 132             if dir:
 133                 if dir.group(2) == build['version_major']:
 134                     snapshot_dir = tag['href']
 135                     build['version_minor'] = dir.group(3)
 136
 137     try:
 138         req = requests.get(snapshot_dir)
 139         req.raise_for_status()
 140     except HTTPError:
 141         print "Could not find the snapshot directory"
 142     else:
 143         urlpath = urlopen(snapshot_dir)
 144         content = urlpath.read().decode('utf-8')
 145         html_content = BeautifulSoup(content, 'html.parser')
 146         # Loops through all the files present in `snapshot_dir`
 147         # and stores the url of latest tarball because files are
 148         # already sorted in early to late order.
 149         for tag in html_content.find_all('a', href=True):
 150             if tag['href'].endswith('tar.gz'):
 151                 snapshot_url = tag['href']
 152
 153         # Get download_url
 154         build['download_url'] = snapshot_url
 155
 156         # Get changelog_date from the snapshot URL
 157         # eg: 'distribution-karaf-0.5.2-20161202.230609-363.tar.gz'
 158         # '\d{8}' searches for the date in the url
 159         extract_date = re.search(r'\d{8}', snapshot_url)
 160         extract_date = extract_date.group(0)
 161         year = int(extract_date[:4])
 162         month = int(extract_date[4:6])
 163         date = int(extract_date[6:])
 164
 165         # %a: Abbreviated weekday name
 166         # %b: Abbreviated month name
 167         # %d: Zero padded decimal number
 168         # %Y: Year
 169         # `changelog_date` is in the format: 'Sat Dec 10 2016'
 170         # Docs:
 171         # https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
 172         build['changelog_date'] = datetime.date(year,
 173                                                 month,
 174                                                 date).strftime("%a %b %d %Y")
 175
 176         # Assign codename
 177         build['codename'] = "SNAPSHOT"
 178         urlpath.close()
 179
 180         build_rpm(build)
 181
 182
 183 # When run as a script, accept a set of builds and execute them
 184 if __name__ == "__main__":
 185     # Load RPM build variables from a YAML config file
 186     build_vars_path = os.path.join(project_root, "build_vars.yaml")
 187     with open(build_vars_path) as rpm_var_file:
 188         build_vars = yaml.load(rpm_var_file)
 189
 190     # Accept the version(s) of the build(s) to perform as args
 191     # TODO: More docs on ArgParser and argument
 192     parser = argparse.ArgumentParser(conflict_handler='resolve')
 193     existing_build_group = parser.add_argument_group("Existing build")
 194     existing_build_group.add_argument(
 195         "-v", "--version", action="append", metavar="major minor patch rpm",
 196         nargs="*", help="RPM version(s) to build"
 197     )
 198     new_build_group = parser.add_argument_group("New build")
 199     new_build_group.add_argument(
 200         "--major", help="Major (element) version to build")
 201     new_build_group.add_argument("--minor", help="Minor (SR) version to build")
 202     new_build_group.add_argument("--patch", help="Patch version to build")
 203     new_build_group.add_argument("--rpm",   help="RPM version to build")
 204     new_build_group.add_argument(
 205         "--sysd_commit", help="Version of ODL unitfile to package")
 206     new_build_group.add_argument("--codename", help="Codename for ODL version")
 207     new_build_group.add_argument(
 208         "--download_url", help="Tarball to repackage into RPM")
 209     new_build_group.add_argument(
 210         "--changelog_date", help="Date this RPM was defined")
 211     new_build_group.add_argument(
 212         "--changelog_name", help="Name of person who defined RPM")
 213     new_build_group.add_argument(
 214         "--changelog_email", help="Email of person who defined RPM")
 215
 216     # Arguments needed to build RPM from latest snapshot
 217     # given a stable major branch
 218     latest_snap_group = parser.add_argument_group("Latest snapshot build")
 219     latest_snap_group.add_argument("--build-latest-snap", action='store_true',
 220                                    help="Build RPM from the latest snpashot")
 221     latest_snap_group.add_argument("--major", required='true',
 222                                    help="Stable branch from which "
 223                                    "to build the snapshot")
 224     latest_snap_group.add_argument("--minor", help="Minor version of the "
 225                                    "stable branch to build the snapshot")
 226     latest_snap_group.add_argument("--patch", help="Patch version to build")
 227     latest_snap_group.add_argument("--rpm",   help="RPM version to build")
 228     latest_snap_group.add_argument("--sysd_commit",
 229                                    help="Version of ODL unitfile to package")
 230     latest_snap_group.add_argument("--codename",
 231                                    help="Codename for ODL snapshot")
 232     latest_snap_group.add_argument("--changelog_name",
 233                                    help="Name of person who defined RPM")
 234     latest_snap_group.add_argument("--changelog_email",
 235                                    help="Email of person who defined RPM")
 236     # Print help if no arguments are given
 237     if len(sys.argv) == 1:
 238         parser.print_help()
 239         sys.exit(1)
 240
 241     # Parse the given args
 242     args = parser.parse_args()
 243
 244     # Build list of RPM builds to perform
 245     builds = []
 246     if args.version:
 247         # Build a list of requested versions as dicts of version components
 248         versions = []
 249         version_keys = ["version_major", "version_minor", "version_patch",
 250                         "rpm_release"]
 251         # For each version arg, match all version components to build_vars name
 252         for version in args.version:
 253             versions.append(dict(zip(version_keys, version)))
 254
 255         # Find every RPM build that matches any version argument
 256         # A passed version "matches" a build when the provided version
 257         # components are a subset of the version components of a build. Any
 258         # version components that aren't passed are simply not checked, so
 259         # they can't fail the match, effectively wild-carding them.
 260         for build in build_vars["builds"]:
 261             for version in versions:
 262                 # Converts both dicts' key:value pairs to lists of tuples and
 263                 # checks that each tuple in the version list is present in the
 264                 # build list.
 265                 if all(item in build.items() for item in version.items()):
 266                     builds.append(build)
 267     else:
 268         builds.append({"version_major": args.major,
 269                        "version_minor": args.minor,
 270                        "version_patch": args.patch,
 271                        "rpm_release": args.rpm,
 272                        "sysd_commit": args.sysd_commit,
 273                        "codename": args.codename,
 274                        "download_url": args.download_url,
 275                        "changelog_date": args.changelog_date,
 276                        "changelog_name": args.changelog_name,
 277                        "changelog_email": args.changelog_email})
 278
 279     # If the flag `--build-latest-snap` is true, extract information
 280     # from the snapshot URL, else directly build the RPM
 281     for build in builds:
 282         if args.build_latest_snap:
 283             build_snapshot_rpm(build)
 284         else:
 285             build_rpm(build)