From 359ff75b09935750d9976edd6a48cee590cd66d4 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Mon, 18 Jul 2016 19:03:23 -0400 Subject: [PATCH] Remove jjb custom scripting No longer used or necessary to manage releng/builder templates. Change-Id: Ib6faaad9e734c8a46e018b71efa8763ac4ec45c6 Signed-off-by: Thanh Ha --- jjb-templates/project.yaml | 24 ---- scripts/jjb-autoupdate-project.py | 69 ---------- scripts/jjb-init-project.py | 217 ------------------------------ scripts/jjblib.py | 100 -------------- 4 files changed, 410 deletions(-) delete mode 100644 jjb-templates/project.yaml delete mode 100644 scripts/jjb-autoupdate-project.py delete mode 100644 scripts/jjb-init-project.py delete mode 100644 scripts/jjblib.py diff --git a/jjb-templates/project.yaml b/jjb-templates/project.yaml deleted file mode 100644 index a77fd1e5d..000000000 --- a/jjb-templates/project.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# ODL Releng build templates -- project: - name: PROJECT_SHORTNAME - jobs: - - '{name}-clm-{stream}' - - '{name}-distribution-{stream}' - - '{name}-distribution-check-{stream}' - - '{name}-integration-{stream}' - - '{name}-merge-{stream}' - - '{name}-periodic-{stream}' - - '{name}-sonar' - - '{name}-validate-autorelease-{stream}' - - '{name}-verify-{stream}' - - # stream: release stream (eg. stable-lithium or beryllium) - # branch: git branch (eg. stable/lithium or master) - STREAMS - project: 'PROJECT_PATH' - mvn-settings: 'JENKINS_SETTINGS' - mvn-goals: 'MAVEN_GOALS' - mvn-opts: 'MAVEN_OPTS' - dependencies: 'DEPENDENCIES' - email-upstream: 'EMAIL_PREFIX' - archive-artifacts: 'ARCHIVE_ARTIFACTS' diff --git a/scripts/jjb-autoupdate-project.py b/scripts/jjb-autoupdate-project.py deleted file mode 100644 index 56769bdb7..000000000 --- a/scripts/jjb-autoupdate-project.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python - -# @License EPL-1.0 -############################################################################## -# Copyright (c) 2014, 2015 The Linux Foundation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# which accompanies this distribution, and is available at -# http://www.eclipse.org/legal/epl-v10.html -# -# Contributors: -# Thanh Ha (The Linux Foundation) - Initial implementation -############################################################################## - -import os - -from jjblib import Project - - -def _is_project(jjb_dir, project): - """Returns true if project is a project that should be auto-generated""" - p = Project(project) - if p.meta_project: - template = os.path.join(jjb_dir, project, "%s.yaml" % p.project) - else: - template = os.path.join(jjb_dir, p.project, "%s.yaml" % p.project) - - if os.path.isfile(template): # Project found - with open(template, 'r') as f: - first_line = f.readline() - if first_line.startswith("# REMOVE THIS LINE IF"): - return True - return False - - -def get_autoupdate_projects(jjb_dir): - """Get list of projects that should be auto-updated.""" - project_list = [] - - for root, dirs, files in os.walk(jjb_dir): - project = root.replace("%s/" % jjb_dir, '') - - if _is_project(jjb_dir, project): - project_list.append(project) - - return project_list - - -def update_templates(projects): - for project in projects: - print("Updating... %s" % project) - p = Project(project) - if p.meta_project: # Meta project - cfg_file = "jjb/%s/%s.cfg" % (project, p.project) - else: - cfg_file = "jjb/%s/%s.cfg" % (p.project, p.project) - os.system("python scripts/jjb-init-project.py %s -c %s" % - (project, cfg_file)) - -############## -# Code Start # -############## - -jjb_dir = "jjb" -all_projects = [d for d in os.listdir(jjb_dir) - if os.path.isdir(os.path.join(jjb_dir, d))] -auto_update_projects = get_autoupdate_projects(jjb_dir) -update_templates(auto_update_projects) diff --git a/scripts/jjb-init-project.py b/scripts/jjb-init-project.py deleted file mode 100644 index b878f8e4b..000000000 --- a/scripts/jjb-init-project.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/python - -# @License EPL-1.0 -############################################################################## -# Copyright (c) 2014, 2015 The Linux Foundation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Eclipse Public License v1.0 -# which accompanies this distribution, and is available at -# http://www.eclipse.org/legal/epl-v10.html -# -# Contributors: -# Thanh Ha (The Linux Foundation) - Initial implementation -############################################################################## - -from collections import OrderedDict -import os -import re - -import yaml - -import jjblib - - -args = jjblib.parse_jjb_args() - - -project = jjblib.Project(args.project) -if project.meta_project is not None: - project_dir = os.path.join("jjb", project.meta_project, project.project) - jenkins_settings = "%s-%s-settings" % (project.meta_project, - project.project) -else: - project_dir = os.path.join("jjb", project.project) - jenkins_settings = "%s-settings" % project.project - -project_file = os.path.join(project_dir, "%s.yaml" % project) -dependent_jobs = "" -disabled = "true" # Always disabled unless project has dependencies -email_prefix = "[%s]" % project - - -if not args.conf: - jjblib.create_template_config(project_dir, args) - project_conf = os.path.join(project_dir, "%s.cfg" % args.project) -else: - project_conf = args.conf - -cfg = dict() # Needed to skip missing project.cfg files -if os.path.isfile(project_conf): - stream = open(project_conf, "r") - cfg = yaml.load(stream) - -#################### -# Handle Templates # -#################### -if cfg.get("JOB_TEMPLATES"): - templates = cfg.get("JOB_TEMPLATES") -else: - templates = ("") - -################## -# Handle Streams # -################## -streams = OrderedDict() -if cfg.get("STREAMS"): # this is a list of single-key dicts - for stream_dict in cfg.get("STREAMS"): - streams.update(stream_dict) -else: - streams = {"boron": jjblib.STREAM_DEFAULTS["boron"]} - -first_stream = next(iter(streams)) # Keep master branch at top. -sonar_branch = streams[first_stream]["branch"] -# Create YAML to list branches to create jobs for -str_streams = "stream:\n" -for stream, options in streams.items(): - str_streams += (" - %s:\n" - " branch: '%s'\n" % - (stream, options["branch"])) - str_streams += " jdk: %s\n" % options["jdks"].split(',')[0].strip() # noqa - str_streams += " jdks:\n" - for jdk in options["jdks"].split(","): - str_streams += " - %s\n" % jdk.strip() - - # Disable autorelease validate job unless project is participating - # in autorelease, JJB does not allow flipping a boolean so we have to - # flip it here via not operator since the JJB configuration for disabling - # a Jenkins Job is "disabled: bool". - str_streams += " disable_autorelease: %s\n" % (not options.get( - "autorelease", False)) - - # Disable the distribution-check job unless project enables it - str_streams += " disable_distribution_check: %s\n" % ( - not options.get("distribution-check", True)) - -############### -# Handle JDKS # -############### -if cfg.get('JDKS'): - jdks = cfg.get('JDKS') -else: - jdks = "openjdk7" -use_jdks = "" -for jdk in jdks.split(","): - use_jdks += " - %s\n" % jdk - -############## -# Handle POM # -############## -if cfg.get('POM'): - pom = cfg.get('POM') -else: - pom = "pom.xml" - -#################### -# Handle MVN_GOALS # -#################### -if cfg.get('MVN_GOALS'): - mvn_goals = cfg.get('MVN_GOALS') -else: - mvn_goals = ("clean install " - "-Dmaven.repo.local=/tmp/r " - "-Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r ") - -################### -# Handle MVN_OPTS # -################### -if cfg.get('MVN_OPTS'): - mvn_opts = cfg.get('MVN_OPTS') -else: - mvn_opts = "-Xmx1024m -XX:MaxPermSize=256m" - -####################### -# Handle DEPENDENCIES # -####################### -if cfg.get('DEPENDENCIES'): - dependencies = cfg.get('DEPENDENCIES') - if dependencies.find("odlparent") < 0: # Add odlparent if not listed - dependencies = "odlparent," + dependencies - disabled = "false" -else: - dependencies = None - if project.project != "odlparent": # Odlparent does not depend on itself - dependencies = "odlparent" # All other projects depend on odlparent - disabled = "false" - -if dependencies: - email_prefix = (email_prefix + " " + - " ".join(['[%s]' % d for d in dependencies.split(",")])) # noqa - dependent_jobs = ",".join( - ['%s-merge-{stream}' % d for d in dependencies.split(",")]) - -############################ -# Handle ARCHIVE_ARTIFACTS # -############################ - -archive_artifacts = cfg.get('ARCHIVE_ARTIFACTS', '') - -############################## -# Create configuration start # -############################## - -# Create project directory if it doesn't exist -if not os.path.exists(project_dir): - os.makedirs(project_dir) - -print("project: %s\n" - "streams: %s\n" - "goals: %s\n" - "options: %s\n" - "dependencies: %s\n" - "artifacts: %s" % - (project, - str_streams, - mvn_goals, - mvn_opts, - dependencies, - archive_artifacts,)) - -# Create initial project YAML file -use_templates = templates.split(",") -use_templates.insert(0, "project") -job_templates_yaml = "" -for t in use_templates: - if t == "project": # This is not a job type but is used for templating - pass - elif t == "sonar": - job_templates_yaml = job_templates_yaml + \ - " - '%s-%s'\n" % (project, t) - else: - job_templates_yaml = job_templates_yaml + \ - " - '%s-%s-{stream}'\n" % (project, t) - -with open(project_file, "w") as outfile: - for t in use_templates: - template_file = "jjb-templates/%s.yaml" % t - with open(template_file, "r") as infile: - for line in infile: - if not re.match("\s*#", line): - line = re.sub("JOB_TEMPLATES", job_templates_yaml, line) - line = re.sub("PROJECT_SHORTNAME", project.project, line) - line = re.sub("PROJECT_PATH", project.path, line) - line = re.sub("JENKINS_SETTINGS", jenkins_settings, line) - line = re.sub("DISABLED", disabled, line) - line = re.sub("STREAMS", str_streams, line) - line = re.sub("POM", pom, line) - line = re.sub("MAVEN_GOALS", mvn_goals, line) - line = re.sub("MAVEN_OPTS", mvn_opts, line) - line = re.sub("DEPENDENCIES", dependent_jobs, line) - line = re.sub("EMAIL_PREFIX", email_prefix, line) - line = re.sub("SONAR_BRANCH", sonar_branch, line) - line = re.sub("ARCHIVE_ARTIFACTS", archive_artifacts, line) - # The previous command may have created superfluous lines. - # If a line has no non-whitespace, it has to be '\n' only. - line = re.sub(r'^\s+\n', "", line) - outfile.write(line) - outfile.write("\n") diff --git a/scripts/jjblib.py b/scripts/jjblib.py deleted file mode 100644 index f48ab9166..000000000 --- a/scripts/jjblib.py +++ /dev/null @@ -1,100 +0,0 @@ -import argparse -import collections -import os - -import yaml - - -def parse_jjb_args(): - parser = argparse.ArgumentParser() - parser.add_argument("project", help="project") - parser.add_argument("-c", "--conf", help="Config file") - parser.add_argument("-d", "--dependencies", - help=("Project dependencies\n\n" - "A comma-seperated (no spaces) list of projects " - "your project depends on. " - "This is used to create an integration job that " - "will trigger when a dependent project-merge " - "job is built successfully.\n\n" - "Example: aaa,controller,yangtools")) - parser.add_argument("-t", "--templates", help="Job templates to use") - parser.add_argument("-s", "--streams", - help="Release streams to fill with default options") - parser.add_argument("-p", "--pom", help="Path to pom.xml to use in Maven " - "build (Default: pom.xml") - parser.add_argument("-g", "--mvn-goals", help="Maven Goals") - parser.add_argument("-o", "--mvn-opts", help="Maven Options") - parser.add_argument("-a", "--archive-artifacts", - help="Comma-seperated list of patterns of artifacts " - "to archive on build completion. " - "See: http://ant.apache.org/manual/Types/fileset.html") # noqa - return parser.parse_args() - - -STREAM_DEFAULTS = collections.OrderedDict([ - ("boron", {"branch": "master", "jdks": "openjdk8"}), - ("beryllium", {"branch": "stable/beryllium", "jdks": "openjdk8"}), - ("stable-lithium", {"branch": "stable/lithium", "jdks": "openjdk7"}), -]) - - -def create_template_config(project_dir, args): - cfg_data = dict() - - if args.templates: - cfg_data["JOB_TEMPLATES"] = args.templates - - if args.streams: - stream_list = list() - for stream in args.streams.split(","): - stream_list.append({stream: STREAM_DEFAULTS[stream]}) - cfg_data["STREAMS"] = stream_list - - if args.pom: - cfg_data["POM"] = args.pom - - if args.mvn_goals: - cfg_data["MAVEN_GOALS"] = args.mvn_goals - - if args.mvn_opts: - cfg_data["MAVEN_OPTS"] = args.mvn_opts - - if args.dependencies: - cfg_data["DEPENDENCIES"] = args.dependencies - - if args.archive_artifacts: - cfg_data["ARCHIVE"] = args.archive_artifacts - - if cfg_data: - # Create project directory if it doesn't exist - if not os.path.exists(project_dir): - os.makedirs(project_dir) - - print("Creating %s.cfg file" % args.project) - cfg_file = os.path.join(project_dir, "%s.cfg" % args.project) - - with open(cfg_file, "w") as outstream: - outstream.write(yaml.dump(cfg_data, default_flow_style=False)) - - -class Project: - """Represents a Gerrit Project - - Attributes: - path(str): The full Gerrit path to a project - meta_project(str): The top-level project name in Gerrit - project(str): The subproject name or project shortname - """ - - def __init__(self, project): - self.path = project - self.meta_project = None - self.project = project - - if project.find('/') >= 0: - s = project.rsplit('/', 1) - self.meta_project = s[0] - self.project = s[1] - - def __str__(self): - return self.project -- 2.36.6