From df2294ec116337450d476d84d11cb2cdbbc79613 Mon Sep 17 00:00:00 2001 From: Tomas Cere Date: Thu, 19 Aug 2021 13:11:07 +0200 Subject: [PATCH] Add mri-stage job This job stages a project and then triggers an mri distribution test job which runs csit on the staged artifacts from the staging repo. Add netconf stage/distribution job definitions Repoint netconf streams to master Change-Id: I473ef076e22df17bb5e0c0f7a9e7adcac0bd2822 Signed-off-by: Tomas Cere --- jjb/defaults.yaml | 14 +- jjb/integration/integration-test-jobs.yaml | 33 +++ jjb/netconf/netconf-clustering-scale.yaml | 2 +- jjb/netconf/netconf-clustering.yaml | 2 +- jjb/netconf/netconf-csit-callhome.yaml | 2 +- .../netconf-distribution-mri-test.yaml | 30 +++ jjb/netconf/netconf-patch-test.yaml | 2 +- jjb/netconf/netconf-scale.yaml | 2 +- jjb/netconf/netconf-userfeatures.yaml | 2 +- jjb/netconf/netconf.yaml | 4 + jjb/releng-maven-mri-stage.sh | 33 +++ jjb/releng-maven-mri-stage.yaml | 206 ++++++++++++++++++ jjb/releng-templates-java.yaml | 10 + 13 files changed, 329 insertions(+), 13 deletions(-) create mode 100644 jjb/netconf/netconf-distribution-mri-test.yaml create mode 100644 jjb/releng-maven-mri-stage.sh create mode 100644 jjb/releng-maven-mri-stage.yaml diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml index 63b6f381e7..b227f087c1 100644 --- a/jjb/defaults.yaml +++ b/jjb/defaults.yaml @@ -278,13 +278,13 @@ controller-csit-3node-rest-clust-cars-perf-ask-only-phosphorus, controller-csit-3node-rest-clust-cars-perf-tell-only-phosphorus, mdsal-csit-3node-netty-replicate-only-phosphorus, - netconf-csit-1node-callhome-only-phosphorus, - netconf-csit-1node-scale-max-devices-only-phosphorus, - netconf-csit-1node-scale-only-phosphorus, - netconf-csit-1node-userfeatures-all-phosphorus, - netconf-csit-1node-userfeatures-rfc8040-all-phosphorus, - netconf-csit-3node-clustering-only-phosphorus, - netconf-csit-3node-clustering-scale-only-phosphorus, + netconf-csit-1node-callhome-only-master, + netconf-csit-1node-scale-max-devices-only-master, + netconf-csit-1node-scale-only-master, + netconf-csit-1node-userfeatures-all-master, + netconf-csit-1node-userfeatures-rfc8040-all-master, + netconf-csit-3node-clustering-only-master, + netconf-csit-3node-clustering-scale-only-master, yangtools-csit-1node-system-only-phosphorus csit-weekly-list-phosphorus: > controller-csit-1node-notifications-longevity-only-phosphorus, diff --git a/jjb/integration/integration-test-jobs.yaml b/jjb/integration/integration-test-jobs.yaml index 1ea80ad64e..0baa6db296 100644 --- a/jjb/integration/integration-test-jobs.yaml +++ b/jjb/integration/integration-test-jobs.yaml @@ -500,6 +500,39 @@ email-recipients: "{email-recipients}" email-prefix: "[integration]" +- job-template: + name: "{project}-distribution-mri-test-{stream}" + id: inttest-distribution-mri-test + + project-type: freestyle + node: queue-intque-2c-1g + + properties: + - opendaylight-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + parameters: + - integration-bundle-url: + bundle-url: "{bundle-url}" + - distribution-karaf-version: + karaf-version: "{karaf-version}" + + triggers: + - timed: "{schedule-weekly}" + + builders: + - trigger-builds: + - project: "{csit-mri-list}" + block: true + predefined-parameters: | + BUNDLE_URL=$BUNDLE_URL + KARAF_VERSION=$KARAF_VERSION + + publishers: + - email-notification: + email-recipients: "{email-recipients}" + email-prefix: "[integration]" + - job-template: name: "integration-distribution-weekly-test-trigger-{stream}" diff --git a/jjb/netconf/netconf-clustering-scale.yaml b/jjb/netconf/netconf-clustering-scale.yaml index 1fb4d5fafb..49e5a81642 100644 --- a/jjb/netconf/netconf-clustering-scale.yaml +++ b/jjb/netconf/netconf-clustering-scale.yaml @@ -12,7 +12,7 @@ # Project branches stream: - - phosphorus: + - master: branch: "master" jre: "openjdk11" - silicon: diff --git a/jjb/netconf/netconf-clustering.yaml b/jjb/netconf/netconf-clustering.yaml index 3e98cc930d..cdc0e1a057 100644 --- a/jjb/netconf/netconf-clustering.yaml +++ b/jjb/netconf/netconf-clustering.yaml @@ -12,7 +12,7 @@ # Project branches stream: - - phosphorus: + - master: branch: "master" jre: "openjdk11" - silicon: diff --git a/jjb/netconf/netconf-csit-callhome.yaml b/jjb/netconf/netconf-csit-callhome.yaml index 06d26ba611..7f3dd75afb 100644 --- a/jjb/netconf/netconf-csit-callhome.yaml +++ b/jjb/netconf/netconf-csit-callhome.yaml @@ -14,7 +14,7 @@ # Project branches stream: - - phosphorus: + - master: branch: "master" jre: "openjdk11" - silicon: diff --git a/jjb/netconf/netconf-distribution-mri-test.yaml b/jjb/netconf/netconf-distribution-mri-test.yaml new file mode 100644 index 0000000000..de768e63f2 --- /dev/null +++ b/jjb/netconf/netconf-distribution-mri-test.yaml @@ -0,0 +1,30 @@ +--- +- project: + name: netconf-distribution-mri-test + jobs: + - inttest-distribution-mri-test + + project: "netconf" + + stream: + - master: + branch: "master" + java-version: "openjdk11" + - silicon: + branch: "stable/silicon" + java-version: "openjdk11" + - aluminium: + branch: "stable/aluminium" + java-version: "openjdk11" + + os-branch: "queens" + + karaf-version: netconf + + csit-mri-list: > + netconf-csit-1node-gate-callhome-only-{stream}, + netconf-csit-1node-gate-userfeatures-all-{stream}, + netconf-csit-1node-gate-userfeatures-rfc8040-all-{stream}, + netconf-csit-1node-scale-only-{stream}, + netconf-csit-3node-clustering-only-{stream}, + netconf-csit-3node-clustering-scale-only-{stream} diff --git a/jjb/netconf/netconf-patch-test.yaml b/jjb/netconf/netconf-patch-test.yaml index 4b6984a168..5dbe246fa1 100644 --- a/jjb/netconf/netconf-patch-test.yaml +++ b/jjb/netconf/netconf-patch-test.yaml @@ -7,7 +7,7 @@ project: "netconf" stream: - - phosphorus: + - master: branch: "master" java-version: "openjdk11" - silicon: diff --git a/jjb/netconf/netconf-scale.yaml b/jjb/netconf/netconf-scale.yaml index 3f5581c94c..a08f3c2887 100644 --- a/jjb/netconf/netconf-scale.yaml +++ b/jjb/netconf/netconf-scale.yaml @@ -12,7 +12,7 @@ # Project branches stream: - - phosphorus: + - master: branch: "master" jre: "openjdk11" - silicon: diff --git a/jjb/netconf/netconf-userfeatures.yaml b/jjb/netconf/netconf-userfeatures.yaml index 66eab97dd3..1bf4193a7c 100644 --- a/jjb/netconf/netconf-userfeatures.yaml +++ b/jjb/netconf/netconf-userfeatures.yaml @@ -18,7 +18,7 @@ # Project branches stream: - - phosphorus: + - master: branch: "master" jre: "openjdk11" - silicon: diff --git a/jjb/netconf/netconf.yaml b/jjb/netconf/netconf.yaml index ab8f592145..e4f7cab670 100644 --- a/jjb/netconf/netconf.yaml +++ b/jjb/netconf/netconf.yaml @@ -15,6 +15,7 @@ project-name: "netconf" branch: "master" java-version: "openjdk11" + karaf-version: "netconf" mvn-settings: "netconf-settings" mvn-goals: "clean deploy -Pintegrationtests" mvn-opts: "-Xmx2048m -Dmaven.compile.fork=true" @@ -41,6 +42,7 @@ project-name: "netconf" branch: "1.13.x" java-version: "openjdk11" + karaf-version: "netconf" mvn-settings: "netconf-settings" mvn-goals: "clean deploy -Pintegrationtests" mvn-opts: "-Xmx2048m -Dmaven.compile.fork=true" @@ -68,6 +70,7 @@ project-name: "netconf" branch: "stable/aluminium" java-version: "openjdk11" + karaf-version: "netconf" mvn-settings: "netconf-settings" mvn-goals: "clean deploy -Pintegrationtests" mvn-opts: "-Xmx2048m -Dmaven.compile.fork=true" @@ -88,6 +91,7 @@ project: "netconf" project-name: "netconf" branch: "1.7.4.x" + karaf-version: "netconf" mvn-settings: "netconf-settings" mvn-goals: "clean deploy -Pintegrationtests" mvn-opts: "-Xmx2048m -Dmaven.compile.fork=true" diff --git a/jjb/releng-maven-mri-stage.sh b/jjb/releng-maven-mri-stage.sh new file mode 100644 index 0000000000..d3920bb2f0 --- /dev/null +++ b/jjb/releng-maven-mri-stage.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +echo "---> releng-maven-mri-stage.sh" +# This script publishes artifacts to a staging repo in Nexus and exports the karaf bundle URL. +# +# $WORKSPACE/m2repo : Exists and used to deploy the staging repository. +# $NEXUS_URL : Jenkins global variable should be defined. +# $STAGING_PROFILE_ID : Provided by a job parameter. + +# Ensure we fail the job if any steps fail. +set -xeu -o pipefail + +TMP_FILE="$(mktemp)" +lftools deploy nexus-stage "$NEXUS_URL" "$STAGING_PROFILE_ID" "$WORKSPACE/m2repo" | tee "$TMP_FILE" +staging_repo=$(sed -n -e 's/Staging repository \(.*\) created\./\1/p' "$TMP_FILE") + +# Store repo info to a file in archives +mkdir -p "$WORKSPACE/archives" +echo "$staging_repo ${NEXUS_URL}/content/repositories/$staging_repo" | tee -a "$WORKSPACE/archives/staging-repo.txt" + +staged_version=$(find . -name '*karaf*.pom' -exec xpath -q -e '/project/version/text()' {} +) +BUNDLE_URL="${NEXUS_URL}/content/repositories/$staging_repo/org/opendaylight/${PROJECT}/${KARAF_ARTIFACT}/${staged_version}/${KARAF_ARTIFACT}-${staged_version}.zip" + +# Cleanup +rm -f "$TMP_FILE" + +echo "Bundle url is ${BUNDLE_URL}" + +# Re-inject the new BUNDLE_URL for downstream jobs to pull from Nexus +cat > "${WORKSPACE}/maven-staged-bundle.env" < + **/*.log + **/hs_err_*.log + **/target/**/feature.xml + **/target/failsafe-reports/failsafe-summary.xml + **/target/surefire-reports/*-output.txt + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-infra-maven-parameters: + mvn-opts: "{mvn-opts}" + mvn-params: "{mvn-params}" + mvn-version: "{mvn-version}" + - string: + name: ARCHIVE_ARTIFACTS + default: "{archive-artifacts}" + description: Artifacts to archive to the logs server. + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + publishers: + # TODO: Make email notification work. + # - lf-infra-email-notify: + # email-recipients: '{email-recipients}' + # email-prefix: '[releng]' + - lf-infra-publish + +- builder: + name: maven-mri-stage + builders: + - lf-provide-maven-settings: + global-settings-file: "{mvn-global-settings}" + settings-file: "{mvn-settings}" + - lf-infra-create-netrc: + server-id: "{mvn-staging-id}" + # include-raw-escape fails due to JJB bug + - shell: !include-raw-escape: + - global-jjb/shell/common-variables.sh + - releng-maven-mri-stage.sh + - lf-provide-maven-settings-cleanup + +############### +# Maven Stage # +############### + +- releng_maven_mri_stage: &releng_maven_mri_stage + name: releng-maven-mri-stage + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 30 # 30 days in case a release takes long to get approved. + build-timeout: 60 + cron: "@daily" + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + java-version: openjdk8 + maven-versions-plugin: false + version-properties-file: version.properties + mvn-central: false + mvn-global-settings: global-settings + mvn-goals: clean deploy + mvn-opts: "" + mvn-params: "" + mvn-version: mvn35 + ossrh-profile-id: "" + mvn-pom: "" + sign-artifacts: false + sign-mode: serial + stream: master + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + + gerrit_release_triggers: + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+stage-(maven-)?release\s*$' + + gerrit_trigger_file_paths: + - compare-type: ANT + pattern: "**" + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - ".*" + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-infra-maven-parameters: + mvn-opts: "{mvn-opts}" + mvn-params: "{mvn-params}" + mvn-version: "{mvn-version}" + staging-profile-id: "{staging-profile-id}" + - bool: + name: MAVEN_VERSIONS_PLUGIN + default: "{maven-versions-plugin}" + description: Use maven-versions-plugin to update pom versions. + - string: + name: ARCHIVE_ARTIFACTS + default: "{archive-artifacts}" + description: Artifacts to archive to the logs server. + - string: + name: STAGING_PROFILE_ID + default: "{staging-profile-id}" + description: Nexus staging profile ID. + + builders: + - lf-infra-pre-build + - lf-jacoco-nojava-workaround + - integration-set-variables + - lf-maven-install: + mvn-version: "{mvn-version}" + - lf-update-java-alternatives: + java-version: "{java-version}" + - lf-provide-maven-settings: + global-settings-file: "{mvn-global-settings}" + settings-file: "{mvn-settings}" + - lf-maven-versions-plugin: + maven-versions-plugin: "{maven-versions-plugin}" + version-properties-file: "{version-properties-file}" + mvn-version: "{mvn-version}" + mvn-pom: "{mvn-pom}" + mvn-settings: "{mvn-settings}" + - shell: !include-raw-escape: global-jjb/shell/maven-patch-release.sh + - lf-maven-build: + mvn-goals: "{mvn-goals}" + - lf-sigul-sign-dir: + sign-artifacts: "{sign-artifacts}" + sign-dir: "$WORKSPACE/m2repo" + sign-mode: "{sign-mode}" + - maven-mri-stage: + mvn-global-settings: "{mvn-global-settings}" + mvn-settings: "{mvn-settings}" + mvn-staging-id: "{mvn-staging-id}" + - lf-maven-central: + mvn-central: "{mvn-central}" + mvn-global-settings: "{mvn-global-settings}" + mvn-settings: "{mvn-settings}" + ossrh-profile-id: "{ossrh-profile-id}" + - lf-provide-maven-settings-cleanup + - inject: + properties-file: maven-staged-bundle.env + - trigger-builds: + - project: "{project-name}-distribution-mri-test-{stream}" + block: true + predefined-parameters: | + BUNDLE_URL=$BUNDLE_URL + KARAF_VERSION=$KARAF_VERSION + +- job-template: + name: "{project-name}-maven-mri-stage-{stream}" + id: gerrit-maven-mri-stage + <<: *releng_maven_mri_common + # yamllint disable-line rule:key-duplicates + <<: *releng_maven_mri_stage + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default diff --git a/jjb/releng-templates-java.yaml b/jjb/releng-templates-java.yaml index ac491283a2..832db0cc39 100644 --- a/jjb/releng-templates-java.yaml +++ b/jjb/releng-templates-java.yaml @@ -14,6 +14,10 @@ sign-artifacts: true cron: "" + - gerrit-maven-mri-stage: + sign-artifacts: true + cron: "" + java-version: openjdk11 mvn-version: mvn35 @@ -43,6 +47,12 @@ ossrh-profile-id: 6386ebbbf3a563 cron: "" + - gerrit-maven-mri-stage: + sign-artifacts: true + mvn-central: false + ossrh-profile-id: 6386ebbbf3a563 + cron: "" + java-version: openjdk8 mvn-version: mvn35 disable-job: "{disable-job}" -- 2.36.6