X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Freleng-templates-java.yaml;h=87a2dd943e93f579be8cae0bce737d0d298e3c9c;hb=71acdd474603eb01bdd166c7a9deedfdef7fde53;hp=bea4575ae77c376161b80a669e32dfd8e9d2d092;hpb=a4dfeef583abfd000eb90a1d8d6465d6eec886f2;p=releng%2Fbuilder.git diff --git a/jjb/releng-templates-java.yaml b/jjb/releng-templates-java.yaml index bea4575ae..87a2dd943 100644 --- a/jjb/releng-templates-java.yaml +++ b/jjb/releng-templates-java.yaml @@ -12,9 +12,9 @@ node: '{build-node}' jdk: '{jdk}' - logrotate: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' + properties: + - opendaylight-infra-properties: + build-days-to-keep: '{build-days-to-keep}' parameters: - opendaylight-infra-parameters: @@ -40,7 +40,7 @@ global-settings-file: 'odl-global-settings' settings-file: '{mvn-settings}' - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: 'pom.xml' goals: 'clean install dependency:tree com.sonatype.clm:clm-maven-plugin:index -V -B -Pq -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' java-opts: @@ -55,98 +55,29 @@ publishers: - email-notification: + email-recipients: '{email-recipients}' email-prefix: '[{project-name}]' - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' + maven-version: 'mvn33' - job-template: - name: '{project-name}-distribution-{stream}' - - # Required variables: - # stream: release stream (eg. stable-lithium or beryllium) - # branch: git branch (eg. stable/lithium or master) - - project-type: freestyle - node: '{build-node}' - jdk: '{jdk}' - - logrotate: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' - - parameters: - - opendaylight-infra-parameters: - project: 'integration/distribution' - branch: '{branch}' - refspec: 'refs/heads/{branch}' - artifacts: '{archive-artifacts} **/dependency_tree.txt **/target/surefire-reports/*-output.txt' - - scm: - - git-scm: - refspec: '' - branch: '{branch}' - - wrappers: - - opendaylight-infra-wrappers: - build-timeout: '{build-timeout}' - - triggers: - - reverse: - jobs: '{project}-merge-{stream}' - result: 'success' - - builders: - - provide-maven-settings: - global-settings-file: 'odl-global-settings' - settings-file: '{mvn-settings}' - - maven-target: - maven-version: '{mvn33}' - pom: 'pom.xml' - goals: 'clean install dependency:tree -DoutputFile=dependency_tree.txt -V -B -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' - java-opts: - - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' - settings: 'integration-distribution-settings' - settings-type: cfp - global-settings: 'odl-global-settings' - global-settings-type: cfp - - maven-target: - maven-version: '{mvn33}' - pom: 'pom.xml' - goals: 'validate jar:jar deploy:deploy -V -B -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' - java-opts: - - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' - settings: 'integration-distribution-settings' - settings-type: cfp - global-settings: 'odl-global-settings' - global-settings-type: cfp - - publishers: - - email-notification: - email-prefix: '[{project}]' - - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' - -- job-template: - # Template: {name}-distribution-check-{stream} - # Goal: Build a patch and make sure the distribution can deploy with this change + # Template: {project-name}-distribution-check-{stream} + # Goal: Build a patch and make sure the distribution can deploy with this change. # Operation: This job template builds a patch, creates a distribution containing - # the patch, and triggers the distribution deploy test + # the patch (making sure dependencies are specified), + # and performs the distribution deploy test. name: '{project-name}-distribution-check-{stream}' - disabled: '{obj:disable_distribution_check}' + disabled: false project-type: freestyle node: '{build-node}' concurrent: true jdk: '{jdk}' - logrotate: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' + properties: + - opendaylight-infra-properties: + build-days-to-keep: '{build-days-to-keep}' parameters: - opendaylight-infra-parameters: @@ -154,7 +85,8 @@ branch: '{branch}' refspec: 'refs/heads/{branch}' artifacts: '{archive-artifacts} **/dependency_tree.txt **/target/surefire-reports/*-output.txt' - - integration-distribution-git-url + - integration-distribution-git-url: + git-url: '{git-url}' scm: - integration-gerrit-scm: @@ -162,64 +94,156 @@ refspec: '$GERRIT_REFSPEC' branch: '{branch}' - integration-distribution-scm: - branch: '{branch}' + branch: '{distribution_branch}' wrappers: - opendaylight-infra-wrappers: - build-timeout: '{build-timeout}' + # Distro-check jobs typically run within 10 - 30 minutes + # with 45 minutes being the occassional edge case. + # enforce a 60 minute limit to ensure stuck jobs get + # cleared up sooner. + build-timeout: '60' triggers: - - gerrit: - server-name: 'OpenDaylight' - trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'test-distribution' - - comment-added-contains-event: - comment-contains-value: 'reverify' - - comment-added-contains-event: - comment-contains-value: 'recheck' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - skip-vote: - successful: false - failed: false - unstable: false - notbuilt: false + - gerrit-trigger-patch-submitted: + server: '{server-name}' + project: '{project}' + branch: '{branch}' + files: '**' builders: + # Step zero: Wipe file repositories up front. + - shell: | + echo "wipe r: the local Maven repository" + rm -rfv /tmp/r + echo "wipe n: the fake remote (Nexus) repository" + rm -rfv /tmp/n + echo "wipe t: the transient repository used in some manipulations" + rm -rfv /tmp/t + # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/. + # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here. - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: '$GERRIT_PROJECT/pom.xml' - goals: 'clean install dependency:tree -DoutputFile=dependency_tree.txt -V -B -Pq -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r -Dstream={stream}' + goals: 'clean deploy dependency:tree -DoutputFile=dependency_tree.txt -V -B -Pq -Djenkins -Dstream={stream} -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/ -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' java-opts: - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true' settings: '{mvn-settings}' settings-type: cfp global-settings: 'odl-global-settings' global-settings-type: cfp + # Step two: Verify all deployed artifacts belong to the project's groupId. + # This is done by moving the allowed directories out of /tmp/n and checking no files remained there. + # The correct directory is derived from $GERRIT_PROJECT. + - shell: | + mkdir -p /tmp/t/org/opendaylight/$GERRIT_PROJECT + mv /tmp/n/org/opendaylight/$GERRIT_PROJECT/* /tmp/t/org/opendaylight/$GERRIT_PROJECT/ + test `find /tmp/n/ -type f | wc -l` -eq 0 + rm -rf /tmp/n + mv /tmp/t /tmp/n + # Step three: Online build of integration distribution. + # This step is mainly used for downloading other project artifacts. + # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again. - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: 'distribution/pom.xml' - goals: 'clean install dependency:tree -DoutputFile=dependency_tree.txt -V -B -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' + goals: 'clean install dependency:tree -DoutputFile=dependency_tree.txt -V -B -Pq -Djenkins -Dstream={stream} -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' java-opts: - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' settings: '{mvn-settings}' settings-type: cfp global-settings: 'odl-global-settings' global-settings-type: cfp + # Step four: Delete snapshot artifacts from the local repository. + # This is critical to detect orphaned artifacts or missing project-internal dependency declarations. + # Also other files related to maven repository resolution are removed, + # and then empty directories are removed, in order to simplify debugging. + - shell: | + set +e # To avoid failures in projects which generate zero snapshot artifacts. + find /tmp/r/org/opendaylight/$GERRIT_PROJECT/ -path *-SNAPSHOT* -delete + find /tmp/r/ -regex '.*/_remote.repositories\|.*/maven-metadata-local\.xml\|.*/maven-metadata-fake-nexus\.xml\|.*/resolver-status\.properties' -delete + find /tmp/r/ -type d -empty -delete + echo "# INFO: A listing of project related files left in local repository follows." + find /tmp/r/org/opendaylight/$GERRIT_PROJECT/ + true # To prevent the possibly non-zero return code from failing the job. + # Now the ugly part. It seems that the only way to tell Maven 2+ + # which remote repositories to use is via settings.xml file. + # So we create such a file here, but it needs most of odlparent:settings.xml + - shell: | + echo ' + + + + opendaylight-release + + + opendaylight-mirror + opendaylight + https://nexus.opendaylight.org/content/repositories/public/ + never + false + + + + + opendaylight-plugin-mirror + opendaylight-plugin + https://nexus.opendaylight.org/content/repositories/public/ + never + false + + + + + file-snapshots + + + file-snapshots + file + file:///tmp/n/ + false + + + + + file-plugin-snapshots + file-plugin + file:///tmp/n/ + false + + + + + + file-snapshots + opendaylight-release + + + ' > fake_remotes.xml + # # Notes: The settings are minimal in order to detect breakage scenarios while allowing for the following quirks: + # # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org + # # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped. + # # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r + # Step five: Repeat the distribution build but with the new settings. + # Here, only the project snapshot artifacts deployed to /tmp/n are available, + # which faithfully reproduces conditions in later verify-like job runs. + # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode" + # This is where SingleFeatureTest is not skipped. + - maven-target: + maven-version: 'mvn33' + pom: 'distribution/pom.xml' + goals: 'clean install dependency:tree -DoutputFile=dependency_tree.txt -s fake_remotes.xml -V -B -Pq -DskipTests=false -Djenkins -Dstream={stream} -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' + java-opts: + - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true' + # Step six: Run Karaf and verify no critical failures are present. - integration-distribution-check publishers: - email-notification: + email-recipients: '{email-recipients}' email-prefix: '[{project-name}]' - postbuildscript: builders: @@ -233,7 +257,7 @@ - archive: artifacts: '*.zip' - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' + maven-version: 'mvn33' - job-template: name: '{project-name}-integration-{stream}' @@ -245,18 +269,16 @@ # is successfully built. # # Required Variables: - # stream: release stream (eg. stable-lithium or beryllium) - # branch: git branch (eg. stable/lithium or master) + # stream: release stream (eg. boron or carbon) + # branch: git branch (eg. stable/boron or master) project-type: freestyle node: '{build-node}' jdk: '{jdk}' - logrotate: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' + properties: + - opendaylight-infra-properties: + build-days-to-keep: '{build-days-to-keep}' parameters: - opendaylight-infra-parameters: @@ -285,7 +307,7 @@ global-settings-file: 'odl-global-settings' settings-file: '{mvn-settings}' - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' goals: '{mvn-goals} -V -B -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' java-opts: - '{mvn-opts}' @@ -296,11 +318,12 @@ publishers: - email-notification: + email-recipients: '{email-recipients}' email-prefix: '{email-upstream}' - findbugs - jacoco-report - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' + maven-version: 'mvn33' - job-template: name: '{project-name}-merge-{stream}' @@ -311,18 +334,20 @@ # and deploy artifacts to Nexus. # # Required Variables: - # stream: release stream (eg. stable-lithium or beryllium) - # branch: git branch (eg. stable/lithium or master) + # stream: release stream (eg. boron or carbon) + # branch: git branch (eg. stable/boron or master) + # Need to keep jobs that deploy to Nexus at end of build as Maven + # projects. Maybe reconsider this once upstream moves deploy to a + # separate lifecycle: + # https://issues.apache.org/jira/browse/MNG-5666 project-type: maven node: '{build-node}' jdk: '{jdk}' - logrotate: - daysToKeep: '14' - numToKeep: '10' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' + properties: + - opendaylight-infra-properties: + build-days-to-keep: 14 parameters: - opendaylight-infra-parameters: @@ -343,6 +368,7 @@ triggers: - timed: 'H H * * 0' - gerrit-trigger-patch-merged: + server-name: '{server-name}' name: '{project}' branch: '{branch}' @@ -352,8 +378,9 @@ global-settings-file: 'odl-global-settings' settings-file: '{mvn-settings}' + # TODO: Do we want to apply sophisticated checks as in *-distribution-check-*? maven: - maven-name: '{mvn33}' + maven-name: 'mvn33' goals: '{mvn-goals} -V -B -Djenkins -Dmerge -Dstream={stream} -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' maven-opts: '{mvn-opts}' settings: '{mvn-settings}' @@ -376,7 +403,7 @@ # project is configured in a certain way which ODL is not. steps: - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: pom.xml goals: 'site:deploy -V -B -Dstream={stream}' java-opts: @@ -386,7 +413,7 @@ global-settings: 'odl-global-settings' global-settings-type: cfp - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: deploy-site.xml goals: 'site:deploy -V -B -Dstream={stream}' java-opts: @@ -401,6 +428,7 @@ publishers: - email-notification: + email-recipients: '{email-recipients}' email-prefix: '[{project-name}]' - maven-deploy: id: '' @@ -408,68 +436,10 @@ deploy-unstable: false - jacoco-report - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' - -- job-template: - name: '{project-name}-periodic-{stream}' - - # Job template for periodic builders - # - # The purpose of this job template is to setup a periodic - # builder. - # - # Required Variables: - # stream: release stream (eg. stable-lithium or beryllium) - # branch: git branch (eg. stable/lithium or master) - - project-type: freestyle - node: '{build-node}' - jdk: '{jdk}' - - logrotate: - daysToKeep: '14' - - parameters: - - opendaylight-infra-parameters: - project: '{project}' - branch: '{branch}' - refspec: 'refs/heads/{branch}' - artifacts: '{archive-artifacts}' - - scm: - - git-scm: - refspec: '' - branch: '{branch}' - - wrappers: - - opendaylight-infra-wrappers: - build-timeout: '{build-timeout}' - - triggers: - - timed: '@daily' - - builders: - - jacoco-nojava-workaround - - provide-maven-settings: - global-settings-file: 'odl-global-settings' - settings-file: '{mvn-settings}' - - maven-target: - maven-version: '{mvn33}' - goals: '{mvn-goals} -V -B -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' - java-opts: - - '{mvn-opts}' - settings: '{mvn-settings}' - settings-type: cfp - global-settings: 'odl-global-settings' - global-settings-type: cfp - - publishers: - - email-notification: - email-prefix: '[{project-name}]' - - jacoco-report - - findbugs - - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' + maven-version: 'mvn33' + - trigger: + project: 'distribution-merge-{stream}' + threshold: SUCCESS - job-template: name: '{project-name}-sonar' @@ -479,11 +449,9 @@ node: '{build-node}' jdk: 'openjdk8' - logrotate: - daysToKeep: '7' - numToKeep: '10' - artifactDaysToKeep: '1' - artifactNumToKeep: '1' + properties: + - opendaylight-infra-properties: + build-days-to-keep: 7 parameters: - opendaylight-infra-parameters: @@ -504,7 +472,9 @@ triggers: - timed: 'H H * * 6' - gerrit-trigger-patch-sonar: + server-name: '{server-name}' name: '{project}' + # FIXME: Make sure this does not alter Gerrit votes, then update docs. builders: - jacoco-nojava-workaround @@ -512,7 +482,7 @@ global-settings-file: 'odl-global-settings' settings-file: '{mvn-settings}' - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' goals: '{mvn-goals} -V -B -Djenkins -Dsonar -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' maven-opts: - '{mvn-opts}' @@ -521,7 +491,7 @@ global-settings: 'odl-global-settings' global-settings-type: cfp - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' # We should switch to the recommended configuration of sonar once # JJB adds support for configurating the Sonar wrapper: # http://docs.sonarqube.org/display/SCAN/Analyzing+with+SonarQube+Scanner+for+Jenkins @@ -535,23 +505,25 @@ publishers: - email-notification: + email-recipients: '{email-recipients}' email-prefix: '[{project-name}]' - jacoco-report - findbugs - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' + maven-version: 'mvn33' - job-template: name: '{project-name}-validate-autorelease-{stream}' - disabled: '{obj:disable_autorelease}' + disabled: false project-type: freestyle node: '{build-node}' concurrent: true jdk: '{jdk}' - logrotate: - daysToKeep: '7' + properties: + - opendaylight-infra-properties: + build-days-to-keep: 7 parameters: - opendaylight-infra-parameters: @@ -565,7 +537,7 @@ release-branch: '{branch}' - string: name: CLONE_URL - default: 'ssh://jenkins-$SILO@git.opendaylight.org:29418/releng/autorelease' + default: '{git-url}/releng/autorelease' description: "Autorelease clone URL" scm: @@ -585,36 +557,23 @@ build-timeout: '{build-timeout}' triggers: - - gerrit: - server-name: 'OpenDaylight' - trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - - comment-added-contains-event: - comment-contains-value: 'revalidate' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: '**/*.xml' + - gerrit-trigger-patch-submitted: + server: '{server-name}' + project: '{project}' + branch: '{branch}' + files: '**/*.xml' builders: - jacoco-nojava-workaround - autorelease-checkout-gerrit-patch - autorelease-generate-release-patches + # In a perfect world projects should be releasing separately and we consume them + # via a project that pulls the release bits from each project from Nexus. + # Keep the patches compatible with that ideal, but apply an edit + # to enable building in a single maven reactor afterwards. + - autorelease-fix-relative-paths - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: validate-pom.xml goals: 'clean install dependency:tree -V -B -T1.5C -Pq -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' java-opts: @@ -624,7 +583,7 @@ global-settings: 'odl-global-settings' global-settings-type: cfp - maven-target: - maven-version: '{mvn33}' + maven-version: 'mvn33' pom: 'pom.xml' goals: 'clean validate -V -B -Pq -Djenkins -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r' java-opts: @@ -637,9 +596,10 @@ publishers: - email-notification: + email-recipients: '{email-recipients}' email-prefix: '[autorelease] [{project-name}]' - opendaylight-infra-shiplogs: - maven-version: '{mvn33}' + maven-version: 'mvn33' - job-template: name: '{project-name}-verify-{stream}-{maven}-{jdks}' @@ -649,16 +609,17 @@ # The purpose of this job template is to setup a ODL verify job # # Required Variables: - # stream: release stream (eg. stable-lithium or beryllium) - # branch: git branch (eg. stable/lithium or master) + # stream: release stream (eg. boron or carbon) + # branch: git branch (eg. stable/boron or master) project-type: freestyle node: '{build-node}' concurrent: true jdk: '{jdks}' - logrotate: - daysToKeep: '7' + properties: + - opendaylight-infra-properties: + build-days-to-keep: 7 parameters: - opendaylight-infra-parameters: @@ -678,8 +639,10 @@ triggers: - gerrit-trigger-patch-submitted: - name: '{project}' + server: '{server-name}' + project: '{project}' branch: '{branch}' + files: '**' builders: - jacoco-nojava-workaround @@ -699,6 +662,7 @@ publishers: - findbugs - email-notification: + email-recipients: '{email-recipients}' email-prefix: '[{project-name}]' - jacoco-report - opendaylight-infra-shiplogs: