From c7cbc9e7422cc005faa58684ab96a092f0fa2bed Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Tue, 21 Mar 2017 18:51:16 +0100 Subject: [PATCH] Bug 7974: Create check job for distribution The new jobs uses some steps of the usual {project}-distribution-check-{stream} which are now defined as macros. Change-Id: I3ab85ba932b382c793bd5a713e8ee813cecd566c Signed-off-by: Vratko Polak --- .../distribution/distribution-check.yaml | 78 ++++++++ .../distribution/distribution.yaml | 1 + jjb/releng-macros.yaml | 177 ++++++++++++++++++ jjb/releng-templates-java.yaml | 152 +-------------- 4 files changed, 266 insertions(+), 142 deletions(-) create mode 100644 jjb/integration/distribution/distribution-check.yaml diff --git a/jjb/integration/distribution/distribution-check.yaml b/jjb/integration/distribution/distribution-check.yaml new file mode 100644 index 000000000..804c463c1 --- /dev/null +++ b/jjb/integration/distribution/distribution-check.yaml @@ -0,0 +1,78 @@ +--- +- job-template: + # Template: distribution-check-{stream} + # Goal: Build a patch and make sure it would not break distribution-check jobs for other projects. + # Operation: FIXME + # This job template builds a patch, creates a distribution containing + # the patch (making sure dependencies are specified), + # and performs the distribution deploy test. + + name: 'distribution-check-{stream}' + disabled: false + + project-type: freestyle + node: '{build-node}' + concurrent: true + jdk: '{jdk}' + + properties: + - opendaylight-infra-properties: + build-days-to-keep: '{build-days-to-keep}' + + parameters: + - opendaylight-infra-parameters: + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + artifacts: '{archive-artifacts} **/dependency_tree.txt **/target/surefire-reports/*-output.txt' + + scm: + - integration-gerrit-scm: + basedir: 'distribution' + refspec: '$GERRIT_REFSPEC' + branch: '{branch}' + + wrappers: + - opendaylight-infra-wrappers: + # Distro-check jobs typically run within 10 - 30 minutes + # with 45 minutes being the occassional edge case. + # enforce a 60 minute limit to ensure stuck jobs get + # cleared up sooner. + build-timeout: '60' + + triggers: + - gerrit-trigger-patch-submitted: + server: '{server-name}' + project: '{project}' + branch: '{branch}' + files: '**' + + builders: + - distribution-check-wipe + - distribution-check-build-project: + pom: 'distribution/pom.xml' + - distribution-check-verify-groupid: + gerrit-project: 'integration' + - distribution-check-delete-snapshots + - distribution-check-configure-remotes + - distribution-check-repeat-build: + dist-pom: 'distribution/pom.xml' + - integration-distribution-check + + publishers: + - email-notification: + email-recipients: '{email-recipients}' + email-prefix: '[{project-name}]' + - postbuildscript: + builders: + - shell: | + #!/bin/bash + mkdir -p $WORKSPACE/archives + cp karaf*.log $WORKSPACE/archives + script-only-if-succeeded: false + script-only-if-failed: false + mark-unstable-if-failed: true + - archive: + artifacts: '*.zip' + - opendaylight-infra-shiplogs: + maven-version: 'mvn33' diff --git a/jjb/integration/distribution/distribution.yaml b/jjb/integration/distribution/distribution.yaml index 87112e490..ec32f4485 100644 --- a/jjb/integration/distribution/distribution.yaml +++ b/jjb/integration/distribution/distribution.yaml @@ -6,6 +6,7 @@ - '{project-name}-clm-{stream}' - '{project-name}-sonar' - '{project-name}-validate-autorelease-{stream}' + - distribution-check-{stream} stream: - carbon: diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index b8a62a0a5..d441dd940 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -519,3 +519,180 @@ files: - file-id: 'jjbini' target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini' + +- builder: + name: distribution-check-wipe + # Step zero: Wipe file repositories up front. + builders: + - shell: | + echo "wipe r: the local Maven repository" + rm -rfv /tmp/r + echo "wipe n: the fake remote (Nexus) repository" + rm -rfv /tmp/n + echo "wipe t: the transient repository used in some manipulations" + rm -rfv /tmp/t + +- builder: + name: distribution-check-build-project + # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/. + # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here. + # Arguments: + # pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'. + builders: + - maven-target: + maven-version: 'mvn33' + pom: '{pom}' + goals: > + clean deploy dependency:tree + -DoutputFile=dependency_tree.txt + -V -B -Pq + -Djenkins + -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/ + -Dmaven.repo.local=/tmp/r + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r + java-opts: + - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true' + settings: 'integration-settings' + settings-type: cfp + global-settings: 'odl-global-settings' + global-settings-type: cfp + +- builder: + name: distribution-check-verify-groupid + # Step two: Verify all deployed artifacts belong to the project's groupId. + # This is done by moving the allowed directories out of /tmp/n and checking no files remained there. + # The correct directory is derived from $GERRIT_PROJECT. + # TODO: Verify all deployed artifacts are snapshots. + # Arguments: + # gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'. + builders: + - shell: | + mkdir -p /tmp/t/org/opendaylight/{gerrit-project} + mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/ + test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false ) + rm -rf /tmp/n + mv /tmp/t /tmp/n + +- builder: + name: distribution-check-download-deps + # Step three: Online build of integration distribution. + # This step is mainly used for downloading other project artifacts. + # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again. + # Arguments: + # dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended. + builders: + - maven-target: + maven-version: 'mvn33' + pom: '{dist-pom}' + goals: > + clean install dependency:tree + -DoutputFile=dependency_tree.txt + -V -B -Pq + -Djenkins + -Dmaven.repo.local=/tmp/r + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r + java-opts: + - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' + settings: 'integration-settings' + settings-type: cfp + global-settings: 'odl-global-settings' + global-settings-type: cfp + +- builder: + name: distribution-check-delete-snapshots + # Step four: Delete snapshot artifacts from the local repository. + # This is critical to detect orphaned artifacts or missing project-internal dependency declarations. + # Also other files related to maven repository resolution are removed, + # and then empty directories are removed, in order to simplify debugging. + builders: + - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh + +- builder: + name: distribution-check-configure-remotes + # Now the ugly part. It seems that the only way to tell Maven 2+ + # which remote repositories to use is via settings.xml file. + # So we create such a file here, but it needs most of odlparent:settings.xml + builders: + - shell: | + echo ' + + + + opendaylight-release + + + opendaylight-mirror + opendaylight + https://nexus.opendaylight.org/content/repositories/public/ + never + false + + + + + opendaylight-plugin-mirror + opendaylight-plugin + https://nexus.opendaylight.org/content/repositories/public/ + never + false + + + + + file-snapshots + + + file-snapshots + file + file:///tmp/n/ + false + + + + + file-plugin-snapshots + file-plugin + file:///tmp/n/ + false + + + + + + file-snapshots + opendaylight-release + + + ' > fake_remotes.xml + # Notes: The settings are minimal in order to detect breakage scenarios, + # while allowing for the following quirks: + # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org + # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped. + # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r + +- builder: + name: distribution-check-repeat-build + # Step five: Repeat the distribution build but with the new settings. + # Here, only the project snapshot artifacts deployed to /tmp/n are available, + # which faithfully reproduces conditions in later verify-like job runs. + # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode" + # This is where SingleFeatureTest is not skipped. + # Arguments: + # dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended. + builders: + - maven-target: + maven-version: 'mvn33' + pom: '{dist-pom}' + goals: > + clean install dependency:tree + -DoutputFile=dependency_tree.txt -s fake_remotes.xml + -V -B -Pq + -DskipTests=false + -Djenkins + -Dmaven.repo.local=/tmp/r + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r + java-opts: + - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' diff --git a/jjb/releng-templates-java.yaml b/jjb/releng-templates-java.yaml index 08b29ee7f..f6b172c03 100644 --- a/jjb/releng-templates-java.yaml +++ b/jjb/releng-templates-java.yaml @@ -118,149 +118,17 @@ files: '**' builders: - # Step zero: Wipe file repositories up front. - - shell: | - echo "wipe r: the local Maven repository" - rm -rfv /tmp/r - echo "wipe n: the fake remote (Nexus) repository" - rm -rfv /tmp/n - echo "wipe t: the transient repository used in some manipulations" - rm -rfv /tmp/t - # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/. - # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here. - - maven-target: - maven-version: 'mvn33' + - distribution-check-wipe + - distribution-check-build-project: pom: '$GERRIT_PROJECT/pom.xml' - goals: > - clean deploy dependency:tree - -DoutputFile=dependency_tree.txt - -V -B -Pq - -Djenkins - -Dstream={stream} - -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/ - -Dmaven.repo.local=/tmp/r - -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r - java-opts: - - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true' - settings: '{mvn-settings}' - settings-type: cfp - global-settings: 'odl-global-settings' - global-settings-type: cfp - # Step two: Verify all deployed artifacts belong to the project's groupId. - # This is done by moving the allowed directories out of /tmp/n and checking no files remained there. - # The correct directory is derived from $GERRIT_PROJECT. - - shell: | - mkdir -p /tmp/t/org/opendaylight/$GERRIT_PROJECT - mv /tmp/n/org/opendaylight/$GERRIT_PROJECT/* /tmp/t/org/opendaylight/$GERRIT_PROJECT/ - test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false ) - rm -rf /tmp/n - mv /tmp/t /tmp/n - # Step three: Online build of integration distribution. - # This step is mainly used for downloading other project artifacts. - # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again. - - maven-target: - maven-version: 'mvn33' - pom: 'distribution/pom.xml' - goals: > - clean install dependency:tree - -DoutputFile=dependency_tree.txt - -V -B -Pq - -Djenkins - -Dstream={stream} - -Dmaven.repo.local=/tmp/r - -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r - java-opts: - - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' - settings: '{mvn-settings}' - settings-type: cfp - global-settings: 'odl-global-settings' - global-settings-type: cfp - # Step four: Delete snapshot artifacts from the local repository. - # This is critical to detect orphaned artifacts or missing project-internal dependency declarations. - # Also other files related to maven repository resolution are removed, - # and then empty directories are removed, in order to simplify debugging. - - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh - # Now the ugly part. It seems that the only way to tell Maven 2+ - # which remote repositories to use is via settings.xml file. - # So we create such a file here, but it needs most of odlparent:settings.xml - - shell: | - echo ' - - - - opendaylight-release - - - opendaylight-mirror - opendaylight - https://nexus.opendaylight.org/content/repositories/public/ - never - false - - - - - opendaylight-plugin-mirror - opendaylight-plugin - https://nexus.opendaylight.org/content/repositories/public/ - never - false - - - - - file-snapshots - - - file-snapshots - file - file:///tmp/n/ - false - - - - - file-plugin-snapshots - file-plugin - file:///tmp/n/ - false - - - - - - file-snapshots - opendaylight-release - - - ' > fake_remotes.xml - # # Notes: The settings are minimal in order to detect breakage scenarios while allowing for the following quirks: - # # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org - # # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped. - # # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r - # Step five: Repeat the distribution build but with the new settings. - # Here, only the project snapshot artifacts deployed to /tmp/n are available, - # which faithfully reproduces conditions in later verify-like job runs. - # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode" - # This is where SingleFeatureTest is not skipped. - - maven-target: - maven-version: 'mvn33' - pom: 'distribution/pom.xml' - goals: > - clean install dependency:tree - -DoutputFile=dependency_tree.txt -s fake_remotes.xml - -V -B -Pq - -DskipTests=false - -Djenkins - -Dstream={stream} - -Dmaven.repo.local=/tmp/r - -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r - java-opts: - - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' - # Step six: Run Karaf and verify no critical failures are present. + - distribution-check-verify-groupid: + gerrit-project: '$GERRIT_PROJECT' + - distribution-check-download-deps: + dist-pom: 'distribution/pom.xml' + - distribution-check-delete-snapshots + - distribution-check-configure-remotes + - distribution-check-repeat-build: + dist-pom: 'distribution/pom.xml' - integration-distribution-check publishers: -- 2.36.6