X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Freleng-macros.yaml;h=5be34b1f2bb2441e2ce5eb8e300e421ca77fc2be;hb=321d2beb638ea788003884f7aefdde104464c0cc;hp=91e7fe9eccf08a8391f6e85a96369208e3801e51;hpb=e00bc39f5352d8f51b59c68932c1b5964d83e732;p=releng%2Fbuilder.git diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index 91e7fe9ec..6487231cb 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -1,86 +1,138 @@ # OLD Releng macros - +--- - property: name: opendaylight-infra-properties properties: - - build-discarder: - days-to-keep: '{build-days-to-keep}' + - build-discarder: + days-to-keep: '{build-days-to-keep}' + num-to-keep: 40 + # Need to keep artifacts for at least 1 day as some projects need to + # be able to validate their artifacts and only allowing limited + # number of artifacts could make fast moving jobs lost their + # artifacts before the developers can test it. + artifact-days-to-keep: 1 - parameter: name: opendaylight-infra-parameters parameters: - - string: - name: PROJECT - default: '{project}' - description: 'Parameter to identify an ODL Gerrit project' - - string: - name: ARCHIVE_ARTIFACTS - default: '{artifacts} **/target/surefire-reports/*-output.txt' - description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org' - - string: - name: GERRIT_PROJECT - default: '{project}' - description: "GERRIT_PROJECT parameter if not given by trigger" - - string: - name: GERRIT_BRANCH - default: '{branch}' - description: "JJB configured GERRIT_BRANCH parameter" - - string: - name: GERRIT_REFSPEC - default: '{refspec}' - description: "GERRIT_REFSPEC parameter if not given by trigger" - - string: - name: STACK_NAME - default: '$JOB_NAME-$BUILD_NUMBER' - description: "Used by Heat to generate a unique stack & vm name" + - string: + name: PROJECT + default: '{project}' + description: 'Parameter to identify an ODL Gerrit project' + - string: + name: ARCHIVE_ARTIFACTS + # Before adding more here, beware of https://jira.linuxfoundation.org/browse/RELENG-280 + default: > + {artifacts} + **/target/surefire-reports/*-output.txt + **/target/failsafe-reports/failsafe-summary.xml + **/hs_err_*.log + **/target/feature/feature.xml + # TODO: **/*.hprof is too large, compress with xz first + description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org' + - string: + name: GERRIT_PROJECT + default: '{project}' + description: "GERRIT_PROJECT parameter if not given by trigger" + - string: + name: GERRIT_BRANCH + default: '{branch}' + description: "JJB configured GERRIT_BRANCH parameter" + - string: + name: GERRIT_REFSPEC + default: '{refspec}' + description: | + Gerrit reference to checkout (ex: refs/heads/master, + refs/changes/48/61548/1 where 61548 is Gerrit change ID, 1 is + patch number and 48 is last two digits of change ID) + - string: + name: STACK_NAME + default: '$SILO-$JOB_NAME-$BUILD_NUMBER' + description: "Used by Heat to generate a unique stack & vm name" + - string: + name: OS_CLOUD + default: '{os-cloud}' + description: | + The name of a cloud configuration in clouds.yaml. OS_CLOUD is a + variable name that is significant to openstack client as a + environment variable. Please refer to the documentation for + further details. + https://docs.openstack.org/developer/python-openstackclient/ - parameter: name: build-tag parameters: - - string: - name: BUILD_TAG - default: '' - description: 'Tag in Git to checkout' + - string: + name: BUILD_TAG + default: '' + description: 'Tag in Git to checkout' - parameter: name: controller-version-parameter parameters: - - string: - name: ODL_VERSION - default: '{odl_version}' - description: 'Controller version (for use with openstacks networking_odl project)' + - string: + name: ODL_VERSION + default: '{odl_version}' + description: 'Controller version (for use with openstacks networking_odl project)' - parameter: name: patches-to-build-parameter parameters: - - string: - name: PATCHES_TO_BUILD - default: '' - description: 'csv list of patches in project:changeset format to build distribution with' + - string: + name: PATCHES_TO_BUILD + default: '' + description: | + Ordered list of patches to build in the format of project=changeset (checkout) or + project:changeset (cherry-pick) for example: genius=32/53632/9,netvirt:59/50259/47. + A topic is also allowed, for example: topic=binding-tlc-rpc + +- parameter: + name: build-order-parameter + parameters: + - string: + name: BUILD_ORDER + default: '{build-order}' + description: 'Build order for multipatch test when topic is used' + +- parameter: + name: build-fast-parameter + parameters: + - string: + name: BUILD_FAST + default: '{build-fast}' + description: 'Set true to build fast -Pq' + +- parameter: + name: run-test-parameter + parameters: + - string: + name: RUN_TEST + default: '{run-test}' + description: 'Set true to run test after build' - parameter: name: distribution-branch-to-build-parameter parameters: - - string: - name: DISTRIBUTION_BRANCH_TO_BUILD - default: 'master' - description: 'distribution repo branch to build with' + - string: + name: DISTRIBUTION_BRANCH_TO_BUILD + default: 'master' + description: 'distribution repo branch to build with' - parameter: name: p2zip-parameter parameters: - - string: - name: P2ZIP_URL - default: '' - description: 'Nexus staging profile id' + - string: + name: P2ZIP_URL + default: '' + description: 'Nexus staging profile id' - parameter: name: stage-id-parameter parameters: - - string: - name: STAGING_PROFILE_ID - default: '{stage-id}' - description: 'Nexus staging profile id' + - string: + name: STAGING_PROFILE_ID + default: '{stage-id}' + description: 'Nexus staging profile id' - parameter: name: maven-exec @@ -93,305 +145,307 @@ - scm: name: git-scm scm: - - git: - credentials-id: 'opendaylight-jenkins-ssh' - url: '$GIT_BASE' - refspec: '' - branches: - - 'origin/{branch}' - skip-tag: true - wipe-workspace: true + - git: + credentials-id: 'opendaylight-jenkins-ssh' + url: '$GIT_BASE' + refspec: '' + branches: + - 'origin/{branch}' + skip-tag: true + wipe-workspace: true - scm: name: git-scm-with-submodules scm: - - git: - credentials-id: 'opendaylight-jenkins-ssh' - url: '$GIT_BASE' - refspec: '' - branches: - - 'refs/heads/{branch}' - skip-tag: true - wipe-workspace: true - submodule: - recursive: true + - git: + credentials-id: 'opendaylight-jenkins-ssh' + url: '$GIT_BASE' + refspec: '' + branches: + - 'refs/heads/{branch}' + skip-tag: true + wipe-workspace: true + submodule: + recursive: true - scm: name: gerrit-trigger-scm scm: - - git: - credentials-id: 'opendaylight-jenkins-ssh' - url: '$GIT_BASE' - refspec: '{refspec}' - branches: - - 'origin/$GERRIT_BRANCH' - skip-tag: true - choosing-strategy: '{choosing-strategy}' + - git: + credentials-id: 'opendaylight-jenkins-ssh' + url: '$GIT_BASE' + refspec: '{refspec}' + branches: + - 'origin/$GERRIT_BRANCH' + skip-tag: true + choosing-strategy: '{choosing-strategy}' - wrapper: name: build-timeout wrappers: - - timeout: - type: absolute - timeout: 360 - fail: true + - timeout: + type: absolute + timeout: 360 + fail: true # This is a single macro to use for all jobs who vote on every (relevant) patch set. # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs, # in order to prevent Jenkins from starting only a subset and still voting Verified+1. # Arguments: -# server: name of gerrit server to listen to +# gerrit-server-name: name of gerrit server to listen to # project: pattern to match triggering projects # branch: triggering branch name # files: pattern to match triggering filepaths - trigger: name: gerrit-trigger-patch-submitted triggers: - - gerrit: - server-name: '{server}' - trigger-on: - - patchset-created-event: - exclude-drafts: 'true' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'true' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: 'ANT' - pattern: '{files}' + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck$' + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '{files}' # TODO: Unify argument names across gerrit-trigger-* macros. - trigger: name: gerrit-trigger-patch-merged triggers: - - gerrit: - server-name: '{server-name}' - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: '{name}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true - # Force Jenkins always vote the values it should already have voted - # during the prior verify phase - override-votes: true - gerrit-build-started-verified-value: 1 - gerrit-build-successful-verified-value: 1 - gerrit-build-failed-verified-value: 1 - gerrit-build-unstable-verified-value: 1 - gerrit-build-notbuilt-verified-value: 1 - gerrit-build-started-codereview-value: 0 - gerrit-build-successful-codereview-value: 0 - gerrit-build-failed-codereview-value: 0 - gerrit-build-unstable-codereview-value: 0 - gerrit-build-notbuilt-codereview-value: 0 + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge$' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + # Force Jenkins always vote the values it should already have voted + # during the prior verify phase + override-votes: true + gerrit-build-started-verified-value: 1 + gerrit-build-successful-verified-value: 1 + gerrit-build-failed-verified-value: 1 + gerrit-build-unstable-verified-value: 1 + gerrit-build-notbuilt-verified-value: 1 + gerrit-build-started-codereview-value: 0 + gerrit-build-successful-codereview-value: 0 + gerrit-build-failed-codereview-value: 0 + gerrit-build-unstable-codereview-value: 0 + gerrit-build-notbuilt-codereview-value: 0 # TODO: Unify argument names across gerrit-trigger-* macros. - trigger: name: gerrit-trigger-patch-sonar triggers: - - gerrit: - server-name: '{server-name}' - trigger-on: - - comment-added-contains-event: - comment-contains-value: 'run-sonar' - projects: - - project-compare-type: 'ANT' - project-pattern: '{name}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true - -- publisher: - name: archive-artifacts - publishers: - - archive: - artifacts: '{artifacts}' - allow-empty: true - fingerprint: true - latest-only: true + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'run-sonar' + projects: + - project-compare-type: 'ANT' + project-pattern: '{name}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true - publisher: name: email-notification publishers: - - email-ext: - recipients: '{email-recipients}' - reply-to: '' - content-type: default - subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!' - body: | - $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS: - - Check console output at $BUILD_URL to view the results. - unstable: true - fixed: true - send-to: - - recipients + - email-ext: + recipients: '{email-recipients}' + reply-to: '' + content-type: default + subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!' + body: | + $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS: -- publisher: - name: jacoco-report - publishers: - - jacoco: - exec-pattern: "**/**.exec" - class-pattern: "**/classes" - source-pattern: "**/src/main/java" - exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**" - status-update: true - targets: - - branch: - healthy: 10 - unhealthy: 20 - - method: - healthy: 50 - unhealthy: 40 + Please refer to the logs server URL for console logs when possible + and use the Jenkins Build URL as a last resort. -- publisher: - name: opendaylight-infra-shiplogs - # To archive things the job will need to create a "archives" directory in - # the workspace and this macro will handle copying the contents of the - # archives directory. - # - # Uses the build parameter ARCHIVE_ARTIFACTS if not empty to find files - # to archive. You can pass globstar patterns for example "**/*.xml **/*.log" - # as the archive pattern. This is a space separated list of files to archive. - # - # Also ensure that the workspace is cleaned up at the end of the build. - publishers: - - postbuildscript: - builders: - - shell: | - if [ -d "$WORKSPACE/.venv-openstack" ]; then - source $WORKSPACE/.venv-openstack/bin/activate - openstack --os-cloud rackspace stack delete --yes $STACK_NAME - fi - - shell: !include-raw: include-raw-deploy-archives.sh - - maven-target: - maven-version: '{maven-version}' - pom: '.archives/deploy-archives.xml' - goals: 'clean deploy -V -B -q -Dmaven.repo.local=/tmp/r' - settings: 'jenkins-log-archives-settings' - settings-type: cfp - global-settings: 'odl-global-settings' - global-settings-type: cfp - - description-setter: - regexp: '^Build logs: .*' - # Cleanup after ourselves - - wipe-org-opendaylight-repo - script-only-if-succeeded: False - script-only-if-failed: False - mark-unstable-if-failed: True - - workspace-cleanup: - exclude: - # Do not clean up *.jenkins-trigger files for jobs that use a - # properties file as input for triggering another build. - - '**/*.jenkins-trigger' - fail-build: false + Console Logs URL: + https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + + Jenkins Build URL: + $BUILD_URL + unstable: true + fixed: true + send-to: + - recipients + + +- builder: + # Installs the openstack cli into a virtualenv at /tmp/v/openstack + name: odl-openstack-install + builders: + - shell: !include-raw: odl-openstack-install.sh + +- builder: + name: odl-openstack-check-image-protection + builders: + - shell: !include-raw: odl-openstack-check-image-protection.sh + +- builder: + name: odl-openstack-cleanup-old-images + builders: + - shell: !include-raw: odl-openstack-cleanup-old-images.sh + +- builder: + name: odl-openstack-cleanup-orphaned-nodes + builders: + - shell: !include-raw: odl-openstack-cleanup-orphaned-nodes.sh + +- builder: + name: odl-openstack-cleanup-stale-nodes + builders: + - shell: !include-raw: odl-openstack-cleanup-stale-nodes.sh + +- builder: + name: odl-openstack-cleanup-stale-stacks + builders: + - shell: !include-raw: odl-openstack-cleanup-stale-stacks.sh + +- builder: + name: odl-openstack-cleanup-stale-volumes + builders: + - shell: !include-raw: odl-openstack-cleanup-stale-volumes.sh + +- builder: + # Deploys a maven site to Nexus using lftools nexus-zip command + name: opendaylight-infra-deploy-maven-site + builders: + - lf-provide-maven-settings: + global-settings-file: global-settings + settings-file: '{settings-file}' + - lf-infra-create-netrc: + server-id: opendaylight-site + - shell: !include-raw-escape: + - global-jjb/shell/lftools-install.sh + - opendaylight-infra-deploy-maven-site.sh + - lf-provide-maven-settings-cleanup - builder: name: opendaylight-infra-stack # opendaylight-infra-stack.sh has a required variable {stack-template} that # must be passed into this macro. builders: - - shell: !include-raw: opendaylight-infra-stack.sh - - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh + - shell: !include-raw: opendaylight-infra-stack.sh + - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh - builder: name: distribute-build-url # Place URL of the current run of a build job to a file at given path. builders: - - shell: 'echo "$BUILD_URL" > {path}/build.url' + - shell: | + #!/bin/bash + set +e # DO NOT fail script if command returns non-zero. + + echo "$BUILD_URL" > {path}/build.url + + # DO NOT fail the build if the echo failed. + exit 0 - builder: name: wipe-org-opendaylight-repo builders: - - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi' + - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi' - builder: name: wipe-local-maven-repo builders: - - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi' + - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi' - builder: name: jacoco-nojava-workaround builders: - - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes' + - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes' - builder: name: check-clm builders: - - sonatype-clm: - application-name: '{application-name}' + - sonatype-clm: + application-name: '{application-name}' - builder: name: releng-check-unicode builders: - - shell: | - $WORKSPACE/scripts/check-unicode.sh jjb/ + - shell: | + $WORKSPACE/scripts/check-unicode.sh jjb/ - builder: name: provide-maven-settings builders: - - config-file-provider: - files: - - file-id: '{global-settings-file}' - variable: 'GLOBAL_SETTINGS_FILE' - - file-id: '{settings-file}' - variable: 'SETTINGS_FILE' + - config-file-provider: + files: + - file-id: '{global-settings-file}' + variable: 'GLOBAL_SETTINGS_FILE' + - file-id: '{settings-file}' + variable: 'SETTINGS_FILE' - builder: name: releng-fetch-p2zip-if-necessary builders: - - shell: | - # Cleanup any existing zips and metadata before we download the new update site - rm -f *.zip *.xml - - conditional-step: - condition-kind: strings-match - condition-string1: '$P2ZIP_URL' - condition-string2: '' - condition-basedir: workspace - steps: + - shell: | + # Cleanup any existing zips and metadata before we download the new update site + rm -f *.zip *.xml + - conditional-step: + condition-kind: strings-match + condition-string1: '$P2ZIP_URL' + condition-string2: '' + condition-basedir: workspace + steps: # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT - maven-target: maven-version: '{maven-version}' - goals: 'org.apache.maven.plugins:maven-dependency-plugin:get org.apache.maven.plugins:maven-dependency-plugin:copy -V -B -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip -DoutputDirectory=$WORKSPACE' + goals: > + org.apache.maven.plugins:maven-dependency-plugin:get + org.apache.maven.plugins:maven-dependency-plugin:copy + -V -B + -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip + -DoutputDirectory=$WORKSPACE settings: '{settings}' global-settings: '{global-settings}' - builder: name: releng-generate-p2pom builders: - - shell: !include-raw: include-raw-generate-p2pom.sh + - shell: !include-raw: generate-p2pom.sh - builder: name: releng-update-p2composite-metadata builders: - - shell: !include-raw: include-raw-update-p2composite-metadata.sh - - conditional-step: - condition-kind: file-exists - condition-filename: deploy-composite-repo.xml - condition-basedir: workspace - steps: + - shell: !include-raw: update-p2composite-metadata.sh + - conditional-step: + condition-kind: file-exists + condition-filename: deploy-composite-repo.xml + condition-basedir: workspace + steps: - maven-target: maven-version: '{maven-version}' pom: 'deploy-composite-repo.xml' @@ -399,63 +453,68 @@ settings: '{settings}' global-settings: '{global-settings}' +- publisher: + name: releng-openstack-stack-delete + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - SUCCESS + - UNSTABLE + build-steps: + - shell: | + #!/bin/bash + echo "Deleting $STACK_NAME" + source "/tmp/v/openstack/bin/activate" + openstack stack delete --yes "$STACK_NAME" + mark-unstable-if-failed: false + - builder: name: releng-stage-release builders: - - shell: !include-raw: include-raw-stage-release.sh + - shell: !include-raw: stage-release.sh - wrapper: # This wrapper is required for all jobs as it configures the wrappers # needed by OpenDaylight infra. name: opendaylight-infra-wrappers wrappers: - - mask-passwords - - config-file-provider: - files: - - file-id: rackspace-heat - target: '$HOME/.config/openstack/clouds.yaml' - - timeout: - type: absolute - timeout: '{build-timeout}' - timeout-var: 'BUILD_TIMEOUT' - fail: true - - timestamps - - ssh-agent-credentials: - users: - - 'opendaylight-jenkins-ssh' - - openstack: - single-use: True + - mask-passwords + - config-file-provider: + files: + - file-id: npmrc + target: '$HOME/.npmrc' + - file-id: pipconf + target: '$HOME/.config/pip/pip.conf' + - file-id: clouds-yaml + target: '$HOME/.config/openstack/clouds.yaml' + - timeout: + type: absolute + timeout: '{build-timeout}' + timeout-var: 'BUILD_TIMEOUT' + fail: true + - timestamps + - ssh-agent-credentials: + users: + - 'opendaylight-jenkins-ssh' + - openstack: + single-use: true -- builder: - name: packer-validate - builders: - - config-file-provider: - files: - - file-id: 'packer-cloud-env' - variable: 'CLOUDENV' - - shell: | - cd packer - export PACKER_LOG="yes" && \ - export PACKER_LOG_PATH="packer-validate.log" && \ - packer.io validate -var-file=$CLOUDENV \ - -var-file=../packer/vars/{platform}.json \ - ../packer/templates/{template}.json - -- builder: - name: packer-build - builders: - - shell: | - cd packer - export PACKER_LOG="yes" && \ - export PACKER_LOG_PATH="packer-build.log" && \ - packer.io build -var-file=$CLOUDENV \ - -var-file=../packer/vars/{platform}.json \ - ../packer/templates/{template}.json - -- builder: - # TODO: Verify signature after downloading users public key from a locally created - # repository instead of the public keymesh. This requires a process in place to get ODL - # developers public keys into a local repository without increasing the job thoughput. - name: verify-gpg-signature - builders: - - shell: !include-raw: include-raw-verify-gpg-signatures.sh +- publisher: + name: opendaylight-infra-generate-csit-status-report + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - shell: !include-raw: generate-csit-status-report.sh + mark-unstable-if-failed: true