3 ##############################################################################
4 # Copyright (c) 2016 Daniel Farrell and Others. All rights reserved.
6 # This program and the accompanying materials are made available under the
7 # terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 # and is available at http://www.eclipse.org/legal/epl-v10.html
9 ##############################################################################
16 from urllib2 import urlopen
19 from bs4 import BeautifulSoup
21 from requests.exceptions import HTTPError
24 sys.stderr.write("We recommend using our included Vagrant env.\n")
25 sys.stderr.write("Else, do `pip install -r requirements.txt` in a venv.\n")
29 def extract_version(url):
30 """Determine ODL version information from the ODL tarball build URL
32 :arg str url: URL of the ODL tarball build for building RPMs
35 # Version components will be added below. Patch version is always 0.
36 version = {"version_patch": "0"}
38 # Parse URL to find major and minor versions. Eg:
39 # https://nexus.opendaylight.org/content/repositories/public/org/
40 # opendaylight/integration/distribution-karaf/0.3.4-Lithium-SR4/
41 # distribution-karaf-0.3.4-Lithium-SR4.tar.gz
44 re_out = re.search(r'\d\.(\d)\.(\d)', url)
45 version["version_major"] = re_out.group(1)
46 version["version_minor"] = re_out.group(2)
48 # Add version components that need to be extracted based on type of build
49 if "autorelease" in url:
50 version = extract_autorelease_version(url, version)
51 elif "snapshot" in url:
52 version = extract_snapshot_version(url, version)
53 elif "public" or "opendaylight.release" in url:
54 version = extract_release_version(url, version)
56 raise ValueError("Unrecognized URL {}".format(url))
61 def extract_release_version(url, version):
62 """Extract package version components from release build URL
64 :arg str url: URL to release tarball
65 :arg dict version: Package version components for given distro
66 :return dict version: Version components, with additions
68 # If this version of ODL has a codename, parse it from URL. Eg:
69 # https://nexus.opendaylight.org/content/repositories/public/org/
70 # opendaylight/integration/distribution-karaf/0.3.4-Lithium-SR4/
71 # distribution-karaf-0.3.4-Lithium-SR4.tar.gz
72 # codename = Lithium-SR4
73 if int(version["version_major"]) < 7:
74 # ODL versions before Nitrogen use Karaf 3, have a codename
75 # Include "-" in codename to avoid hanging "-" when no codename
76 version["codename"] = "-" + re.search(r'0\.[0-9]+\.[0-9]+-(.*)\/',
79 # ODL versions Nitrogen and after use Karaf 4, don't have a codename
80 version["codename"] = ""
82 # Package version is assumed to be 1 for release builds
83 # TODO: Should be able to manually set this in case this is a rebuild
84 version["pkg_version"] = "1"
89 def extract_autorelease_version(url, version):
90 """Extract package version components from an autorelease build URL
92 :arg str url: URL to autorelease tarball
93 :arg dict version: Package version components for given distro
94 :return dict version: Version components, with additions
96 # If this version of ODL has a codename, parse it from URL. Eg:
97 # https://nexus.opendaylight.org/content/repositories/autorelease-1533/
98 # org/opendaylight/integration/distribution-karaf/0.4.4-Beryllium-SR4/
99 # codename = Beryllium-SR4
100 if int(version["version_major"]) < 7:
101 # ODL versions before Nitrogen use Karaf 3, have a codename
102 # Include "-" in codename to avoid hanging "-" when no codename
103 version["codename"] = "-" + re.search(r'0\.[0-9]+\.[0-9]+-(.*)\/',
106 # ODL versions Nitrogen and after use Karaf 4, don't have a codename
107 version["codename"] = ""
109 # Autorelease URLs don't include a date, parse HTML to find build date
110 # Strip distro zip/tarball archive part of URL, resulting in base URL
111 base_url = url.rpartition("/")[0]+url.rpartition("/")[1]
112 # Using bash subprocess to parse HTML and find date of build
113 # TODO: Do all of this with Python, don't spawn a bash process
114 # Set base_url as an environment var to pass it to subprocess
115 os.environ["base_url"] = base_url
116 raw_date = subprocess.Popen(
117 "curl -s $base_url | grep tar.gz -A1 | tail -n1 |"
118 "sed \"s/<td>//g\" | sed \"s/\\n//g\" | awk '{print $3,$2,$6}' ",
119 shell=True, stdout=subprocess.PIPE,
120 stdin=subprocess.PIPE).stdout.read().rstrip().strip("</td>")
121 build_date = datetime.datetime.strptime(raw_date, "%d %b %Y").strftime(
124 # Parse URL to find unique build ID. Eg:
125 # https://nexus.opendaylight.org/content/repositories/autorelease-1533/
126 # org/opendaylight/integration/distribution-karaf/0.4.4-Beryllium-SR4/
128 build_id = re.search(r'\/autorelease-([0-9]+)\/', url).group(1)
130 # Combine build date and build ID into pkg_version
131 version["pkg_version"] = "0.1." + build_date + "rel" + build_id
136 def extract_snapshot_version(url, version):
137 """Extract package version components from a snapshot build URL
139 :arg str url: URL to snapshot tarball
140 :arg dict version: Package version components for given distro
141 :return dict version: Version components, with additions
144 # All snapshot builds use SNAPSHOT codename
145 # Include "-" in codename to avoid hanging "-" when no codename
146 version["codename"] = "-SNAPSHOT"
148 # Parse URL to find build date and build ID. Eg:
149 # https://nexus.opendaylight.org/content/repositories/
150 # opendaylight.snapshot/org/opendaylight/integration/
151 # distribution-karaf/0.6.0-SNAPSHOT/
152 # distribution-karaf-0.6.0-20161201.031047-2242.tar.gz
153 # build_date = 20161201
155 re_out = re.search(r'0.[0-9]+\.[0-9]+-([0-9]+)\.[0-9]+-([0-9]+)\.', url)
156 build_date = re_out.group(1)
157 build_id = re_out.group(2)
159 # Combine build date and build ID into pkg_version
160 version["pkg_version"] = "0.1." + build_date + "snap" + build_id
165 def get_snap_url(version_major, version_minor=None):
166 """Fetches tarball url for snapshot releases using version information
168 :arg str version_major: Major version for snapshot build
169 :arg str version_minor: Minor version for snapshot build(optional)
170 :return arg snapshot_url: URL of the snapshot release
172 parent_dir = "https://nexus.opendaylight.org/content/repositories/" \
173 "opendaylight.snapshot/org/opendaylight/integration/{}/" \
174 .format(get_distro_name_prefix(version_major))
176 # If the minor verison is given, get the sub-directory directly
177 # else, find the latest sub-directory
181 sub_dir = '0.' + version_major + '.' + version_minor + '-SNAPSHOT/'
182 snapshot_dir = parent_dir + sub_dir
184 subdir_url = urlopen(parent_dir)
185 content = subdir_url.read().decode('utf-8')
186 all_dirs = BeautifulSoup(content, 'html.parser')
188 # Loops through all the sub-directories present and stores the
189 # latest sub directory as sub-directories are already sorted
190 # in early to late order.
191 for tag in all_dirs.find_all('a', href=True):
192 # Checks if the sub-directory name is of the form
193 # '0.<major_version>.<minor_version>-SNAPSHOT'.
194 dir = re.search(r'\/(\d)\.(\d)\.(\d).(.*)\/', tag['href'])
195 # If the major version matches the argument provided
196 # store the minor version, else ignore.
198 if dir.group(2) == version_major:
199 snapshot_dir = tag['href']
200 version_minor = dir.group(3)
203 req = requests.get(snapshot_dir)
204 req.raise_for_status()
206 print "Could not find the snapshot directory"
208 urlpath = urlopen(snapshot_dir)
209 content = urlpath.read().decode('utf-8')
210 html_content = BeautifulSoup(content, 'html.parser')
211 # Loops through all the files present in `snapshot_dir`
212 # and stores the url of latest tarball because files are
213 # already sorted in early to late order.
214 for tag in html_content.find_all('a', href=True):
215 if tag['href'].endswith('tar.gz'):
216 snapshot_url = tag['href']
220 def get_sysd_commit():
221 """Get latest Int/Pack repo commit hash"""
223 int_pack_repo = "https://github.com/opendaylight/integration-packaging.git"
224 # Get the commit hash at the tip of the master branch
225 args_git = ['git', 'ls-remote', int_pack_repo, "HEAD"]
226 args_awk = ['awk', '{print $1}']
227 references = subprocess.Popen(args_git, stdout=subprocess.PIPE,
229 sysd_commit = subprocess.check_output(args_awk, stdin=references.stdout,
235 def get_java_version(version_major):
236 """Get the java_version dependency for ODL builds
238 :arg str version_major: OpenDaylight major version number
239 :return int java_version: Java version required by given ODL version
241 if int(version_major) < 5:
248 def get_changelog_date(pkg_type):
249 """Get the changelog datetime formatted for the given package type
251 :arg str pkg_type: Type of datetime formatting (rpm, deb)
252 :return str changelog_date: Date or datetime formatted for given pkg_type
254 if pkg_type == "rpm":
255 # RPMs require a date of the format "Day Month Date Year". For example:
257 return datetime.date.today().strftime("%a %b %d %Y")
258 elif pkg_type == "deb":
259 # Debs require both a date and time.
260 # Date must be of the format "Day, Date Month Year". For example:
262 date = datetime.date.today().strftime("%a, %d %b %Y")
263 # Time must be of the format "HH:MM:SS +HHMM". For example:
265 time = datetime.datetime.now(tzlocal.get_localzone()).\
266 strftime("%H:%M:%S %z")
267 return "{} {}".format(date, time)
269 raise ValueError("Unknown package type: {}".format(pkg_type))
272 def get_distro_name_prefix(version_major):
273 """Return Karaf 3 or 4-style distro name prefix based on ODL major version
275 :arg str major_version: OpenDaylight major version umber
276 :return str distro_name_style: Karaf 3 or 4-style distro name prefix
279 if int(version_major) < 7:
280 # ODL versions before Nitrogen use Karaf 3, distribution-karaf- names
281 return "distribution-karaf"
283 # ODL versions Nitrogen and after use Karaf 4, karaf- names