3 ##############################################################################
4 # Copyright (c) 2016 Daniel Farrell and Others. All rights reserved.
6 # This program and the accompanying materials are made available under the
7 # terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 # and is available at http://www.eclipse.org/legal/epl-v10.html
9 ##############################################################################
15 from string import Template
20 from urllib2 import urlopen
23 from bs4 import BeautifulSoup
25 from requests.exceptions import HTTPError
28 sys.stderr.write("We recommend using our included Vagrant env.\n")
29 sys.stderr.write("Else, do `pip install -r requirements.txt` in a venv.\n")
32 # Path to directory for cache artifacts
33 cache_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache")
35 # Templates that can be specialized into common artifact names per-build
36 # NB: Templates can't be concatenated with other Templates or strings, or
37 # cast to strings for concatenation. If they could, we would do elegant
38 # refactoring like concatenating paths to templates here and only calling
39 # Template.substitute in the build_rpm function.
40 distro_template = Template("opendaylight-$version_major.$version_minor."
41 "$version_patch-$pkg_version")
42 unitfile_template = Template("opendaylight-$sysd_commit.service")
43 unitfile_url_template = Template("https://git.opendaylight.org/gerrit/"
44 "gitweb?p=integration/packaging.git;a="
45 "blob_plain;f=packages/unitfiles/"
46 "opendaylight.service;hb=$sysd_commit")
49 def extract_version(url):
50 """Determine ODL version information from the ODL tarball build URL
52 :arg str url: URL of the ODL tarball build for building RPMs
55 # Version components will be added below. Patch version is always 0.
56 version = {"version_patch": "0"}
58 # Parse URL to find major and minor versions. Eg:
59 # https://nexus.opendaylight.org/content/repositories/public/org/
60 # opendaylight/integration/distribution-karaf/0.3.4-Lithium-SR4/
61 # distribution-karaf-0.3.4-Lithium-SR4.tar.gz
64 re_out = re.search(r'\d\.(\d+)\.(\d)', url)
65 version["version_major"] = re_out.group(1)
66 version["version_minor"] = re_out.group(2)
68 # Add version components that need to be extracted based on type of build
69 if "autorelease" in url:
70 version = extract_autorelease_version(url, version)
71 elif "snapshot" in url:
72 version = extract_snapshot_version(url, version)
73 elif "public" or "opendaylight.release" in url:
74 version = extract_release_version(url, version)
76 raise ValueError("Unrecognized URL {}".format(url))
81 def extract_release_version(url, version):
82 """Extract package version components from release build URL
84 :arg str url: URL to release tarball
85 :arg dict version: Package version components for given distro
86 :return dict version: Version components, with additions
88 # If this version of ODL has a codename, parse it from URL. Eg:
89 # https://nexus.opendaylight.org/content/repositories/public/org/
90 # opendaylight/integration/distribution-karaf/0.3.4-Lithium-SR4/
91 # distribution-karaf-0.3.4-Lithium-SR4.tar.gz
92 # codename = Lithium-SR4
93 if int(version["version_major"]) < 7:
94 # ODL versions before Nitrogen use Karaf 3, have a codename
95 # Include "-" in codename to avoid hanging "-" when no codename
96 version["codename"] = "-" + re.search(r'0\.[0-9]+\.[0-9]+-(.*)\/',
99 # ODL versions Nitrogen and after use Karaf 4, don't have a codename
100 version["codename"] = ""
102 # Package version is assumed to be 1 for release builds
103 # TODO: Should be able to manually set this in case this is a rebuild
104 version["pkg_version"] = "1"
109 def extract_autorelease_version(url, version):
110 """Extract package version components from an autorelease build URL
112 :arg str url: URL to autorelease tarball
113 :arg dict version: Package version components for given distro
114 :return dict version: Version components, with additions
116 # If this version of ODL has a codename, parse it from URL. Eg:
117 # https://nexus.opendaylight.org/content/repositories/autorelease-1533/
118 # org/opendaylight/integration/distribution-karaf/0.4.4-Beryllium-SR4/
119 # codename = Beryllium-SR4
120 if int(version["version_major"]) < 7:
121 # ODL versions before Nitrogen use Karaf 3, have a codename
122 # Include "-" in codename to avoid hanging "-" when no codename
123 version["codename"] = "-" + re.search(r'0\.[0-9]+\.[0-9]+-(.*)\/',
126 # ODL versions Nitrogen and after use Karaf 4, don't have a codename
127 version["codename"] = ""
129 # Autorelease URLs don't include a date, parse HTML to find build date
130 # Strip distro zip/tarball archive part of URL, resulting in base URL
131 base_url = url.rpartition("/")[0]+url.rpartition("/")[1]
132 # Using bash subprocess to parse HTML and find date of build
133 # TODO: Do all of this with Python, don't spawn a bash process
134 # Set base_url as an environment var to pass it to subprocess
135 os.environ["base_url"] = base_url
136 raw_date = subprocess.Popen(
137 "curl -s $base_url | grep tar.gz -A1 | tail -n1 |"
138 "sed \"s/<td>//g\" | sed \"s/\\n//g\" | awk '{print $3,$2,$6}' ",
139 shell=True, stdout=subprocess.PIPE,
140 stdin=subprocess.PIPE).stdout.read().rstrip().strip("</td>")
141 build_date = datetime.datetime.strptime(raw_date, "%d %b %Y").strftime(
144 # Parse URL to find unique build ID. Eg:
145 # https://nexus.opendaylight.org/content/repositories/autorelease-1533/
146 # org/opendaylight/integration/distribution-karaf/0.4.4-Beryllium-SR4/
148 build_id = re.search(r'\/autorelease-([0-9]+)\/', url).group(1)
150 # Combine build date and build ID into pkg_version
151 version["pkg_version"] = "0.1." + build_date + "rel" + build_id
156 def extract_snapshot_version(url, version):
157 """Extract package version components from a snapshot build URL
159 :arg str url: URL to snapshot tarball
160 :arg dict version: Package version components for given distro
161 :return dict version: Version components, with additions
164 # All snapshot builds use SNAPSHOT codename
165 # Include "-" in codename to avoid hanging "-" when no codename
166 version["codename"] = "-SNAPSHOT"
168 # Parse URL to find build date and build ID. Eg:
169 # https://nexus.opendaylight.org/content/repositories/
170 # opendaylight.snapshot/org/opendaylight/integration/
171 # distribution-karaf/0.6.0-SNAPSHOT/
172 # distribution-karaf-0.6.0-20161201.031047-2242.tar.gz
173 # build_date = 20161201
175 re_out = re.search(r'0.[0-9]+\.[0-9]+-([0-9]+)\.[0-9]+-([0-9]+)\.', url)
176 build_date = re_out.group(1)
177 build_id = re_out.group(2)
179 # Combine build date and build ID into pkg_version
180 version["pkg_version"] = "0.1." + build_date + "snap" + build_id
185 def get_snap_url(version_major):
186 """Get the most recent snapshot build of the given ODL major version
188 :arg str version_major: ODL major version to get latest snapshot of
189 :return str snapshot_url: URL to latest snapshot tarball of ODL version
192 # Dir that contains all shapshot build dirs, varies based on Karaf 3/4
193 parent_dir_url = "https://nexus.opendaylight.org/content/repositories/" \
194 "opendaylight.snapshot/org/opendaylight/integration/{}/" \
195 .format(get_distro_name_prefix(version_major))
197 # Get HTML of dir that contains all shapshot dirs
198 parent_dir_html = urlopen(parent_dir_url).read().decode('utf-8')
200 # Get most recent minor version of the given major version
201 version_minor = max(re.findall(
202 r'>\d\.{}\.(\d)-SNAPSHOT\/'.format(version_major),
205 # Dir that contains snapshot builds for the given major version
206 snapshot_dir_url = parent_dir_url + "0.{}.{}-SNAPSHOT/".format(
210 # Get HTML of dir that contains snapshot builds for given major version
211 snapshot_dir_html = urlopen(snapshot_dir_url).read().decode('utf-8')
213 # Find most recent URL to tarball, ie most recent snapshot build
214 return re.findall(r'href="(.*\.tar\.gz)"', snapshot_dir_html)[-1]
217 def get_sysd_commit():
218 """Get latest Int/Pack repo commit hash"""
220 int_pack_repo = "https://github.com/opendaylight/integration-packaging.git"
221 # Get the commit hash at the tip of the master branch
222 args_git = ['git', 'ls-remote', int_pack_repo, "HEAD"]
223 args_awk = ['awk', '{print $1}']
224 references = subprocess.Popen(args_git, stdout=subprocess.PIPE,
226 sysd_commit = subprocess.check_output(args_awk, stdin=references.stdout,
232 def get_java_version(version_major):
233 """Get the java_version dependency for ODL builds
235 :arg str version_major: OpenDaylight major version number
236 :return int java_version: Java version required by given ODL version
238 if int(version_major) < 5:
245 def get_changelog_date(pkg_type):
246 """Get the changelog datetime formatted for the given package type
248 :arg str pkg_type: Type of datetime formatting (rpm, deb)
249 :return str changelog_date: Date or datetime formatted for given pkg_type
251 if pkg_type == "rpm":
252 # RPMs require a date of the format "Day Month Date Year". For example:
254 return datetime.date.today().strftime("%a %b %d %Y")
255 elif pkg_type == "deb":
256 # Debs require both a date and time.
257 # Date must be of the format "Day, Date Month Year". For example:
259 date = datetime.date.today().strftime("%a, %d %b %Y")
260 # Time must be of the format "HH:MM:SS +HHMM". For example:
262 time = datetime.datetime.now(tzlocal.get_localzone()).\
263 strftime("%H:%M:%S %z")
264 return "{} {}".format(date, time)
266 raise ValueError("Unknown package type: {}".format(pkg_type))
269 def get_distro_name_prefix(version_major, download_url=""):
270 """Return distro name prefix based on ODL major version or distro URL.
272 :arg str version_major: OpenDaylight major version number
273 :arg str download_url: URL to ODL distribution
274 :return str distro_prefix: MR, Karaf 3 or 4-style distro name prefix
277 mrel_prefix = "opendaylight"
278 k3_prefix = "distribution-karaf"
280 mrel_url_base = "https://nexus.opendaylight.org/content/repositories/public/org/opendaylight/integration/opendaylight/"
281 k3_url_base = "https://nexus.opendaylight.org/content/repositories/public/org/opendaylight/integration/distribution-karaf/"
282 k4_url_base = "https://nexus.opendaylight.org/content/repositories/public/org/opendaylight/integration/karaf/"
284 if mrel_url_base in download_url:
286 elif k3_url_base in download_url:
288 elif k4_url_base in download_url:
291 if int(version_major) < 7:
292 # ODL versions before Nitrogen use Karaf 3, distribution-karaf- names
295 # ODL versions Nitrogen and after use Karaf 4, karaf- names
299 def cache_distro(build):
300 """Cache the OpenDaylight distribution to package as RPM/Deb.
302 :param build: Description of an RPM build
304 :return str distro_tar_path: Path to cached distribution tarball
307 # Specialize templates for the given build
308 distro = distro_template.substitute(build)
310 # Append file extensions to get ODL distro zip/tarball templates
311 distro_tar = distro + ".tar.gz"
312 distro_zip = distro + ".zip"
314 # Prepend cache dir path to get template of full path to cached zip/tarball
315 distro_tar_path = os.path.join(cache_dir, distro_tar)
316 distro_zip_path = os.path.join(cache_dir, distro_zip)
318 # Cache OpenDaylight tarball to be packaged
319 if not os.path.isfile(distro_tar_path):
320 if build["download_url"].endswith(".tar.gz"):
321 print("Downloading: {}".format(build["download_url"]))
322 urllib.urlretrieve(build["download_url"], distro_tar_path)
323 print("Cached: {}".format(distro_tar))
324 # If download_url points at a zip, repackage as a tarball
325 elif build["download_url"].endswith(".zip"):
326 if not os.path.isfile(distro_zip):
327 print("URL is to a zip, will download and convert to tar.gz")
328 print("Downloading: {}".format(build["download_url"]))
329 urllib.urlretrieve(build["download_url"], distro_zip_path)
330 print("Downloaded {}".format(distro_zip_path))
332 print("Already cached: {}".format(distro_zip_path))
333 # Extract zip archive
334 # NB: zipfile.ZipFile.extractall doesn't preserve permissions
335 # https://bugs.python.org/issue15795
336 subprocess.call(["unzip", "-oq", distro_zip_path, "-d", cache_dir])
337 # Get files in cache dir
338 cache_dir_ls_all = glob.glob(os.path.join(cache_dir, "*"))
339 # Remove pyc files that may be newer than just-extracted zip
340 cache_dir_ls = filter(lambda f: '.pyc' not in f, cache_dir_ls_all)
341 # Get the most recent file in cache dir, hopefully unzipped archive
342 unzipped_distro_path = max(cache_dir_ls, key=os.path.getctime)
343 print("Extracted: {}".format(unzipped_distro_path))
344 # Remove path from 'unzipped_distro_path', as will cd to dir below
345 unzipped_distro = os.path.basename(unzipped_distro_path)
346 # Using the full paths here creates those paths in the tarball,
347 # which breaks the build. There's a way to change the working dir
348 # during a single tar command using the system tar binary, but I
349 # don't see a way to do that with Python.
350 # TODO: Can this be done without changing directories?
351 # TODO: Try https://goo.gl/XMx5gb
354 with tarfile.open(distro_tar, "w:gz") as tb:
355 tb.add(unzipped_distro)
356 print("Taring {} into {}".format(unzipped_distro, distro_tar))
358 print("Cached: {}".format(distro_tar))
360 print("Already cached: {}".format(distro_tar))
362 return distro_tar_path
365 def cache_sysd(build):
366 """Cache the artifacts required for the given RPM build.
368 :param build: Description of an RPM build
370 :return dict unitfile_path: Paths to cached unit file and unit file tarball
373 # Specialize templates for the given build
374 unitfile = unitfile_template.substitute(build)
375 unitfile_url = unitfile_url_template.substitute(build)
377 # Append file extensions to get ODL distro zip/tarball templates
378 unitfile_tar = unitfile + ".tar.gz"
380 # Prepend cache dir path to get template of full path to cached zip/tarball
381 unitfile_path = os.path.join(cache_dir, unitfile)
382 unitfile_tar_path = os.path.join(cache_dir, unitfile_tar)
384 # Download ODL's systemd unit file
385 if not os.path.isfile(unitfile_path):
386 urllib.urlretrieve(unitfile_url, unitfile_path)
387 print("Cached: {}".format(unitfile))
389 print("Already cached: {}".format(unitfile_path))
391 # Cache ODL's systemd unit file as a tarball
392 if not os.path.isfile(unitfile_tar_path):
393 # Using the full paths here creates those paths in the tarball, which
394 # breaks the build. There's a way to change the working dir during a
395 # single tar command using the system tar binary, but I don't see a
396 # way to do that with Python.
397 # TODO: Is there a good way to do this without changing directories?
398 # TODO: Try https://goo.gl/XMx5gb
401 # Create a .tar.gz archive containing ODL's systemd unitfile
402 with tarfile.open(unitfile_tar, "w:gz") as tb:
406 print("Cached: {}".format(unitfile_tar))
408 print("Already cached: {}".format(unitfile_tar_path))
410 return {"unitfile_tar_path": unitfile_tar_path,
411 "unitfile_path": unitfile_path}