Ensure we fail packer jobs if any step fails
[releng/builder.git] / jjb / releng-macros.yaml
1 # OLD Releng macros
2 ---
3 - property:
4     name: opendaylight-infra-properties
5     properties:
6       - build-discarder:
7           days-to-keep: '{build-days-to-keep}'
8           num-to-keep: 40
9           # Need to keep artifacts for at least 1 day as some projects need to
10           # be able to validate their artifacts and only allowing limited
11           # number of artifacts could make fast moving jobs lost their
12           # artifacts before the developers can test it.
13           artifact-days-to-keep: 1
14
15 - parameter:
16     name: opendaylight-infra-parameters
17     parameters:
18       - string:
19           name: PROJECT
20           default: '{project}'
21           description: 'Parameter to identify an ODL Gerrit project'
22       - string:
23           name: ARCHIVE_ARTIFACTS
24           default: >
25               {artifacts}
26               **/target/surefire-reports/*-output.txt
27               **/hs_err_*.log
28               **/target/feature/feature.xml
29               **/*.hprof
30           description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
31       - string:
32           name: GERRIT_PROJECT
33           default: '{project}'
34           description: "GERRIT_PROJECT parameter if not given by trigger"
35       - string:
36           name: GERRIT_BRANCH
37           default: '{branch}'
38           description: "JJB configured GERRIT_BRANCH parameter"
39       - string:
40           name: GERRIT_REFSPEC
41           default: '{refspec}'
42           description: "GERRIT_REFSPEC parameter if not given by trigger"
43       - string:
44           name: STACK_NAME
45           default: '$SILO-$JOB_NAME-$BUILD_NUMBER'
46           description: "Used by Heat to generate a unique stack & vm name"
47       - string:
48           name: OS_CLOUD
49           default: '{os-cloud}'
50           description: |
51               The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
52               variable name that is significant to openstack client as a
53               environment variable. Please refer to the documentation for
54               further details.
55               https://docs.openstack.org/developer/python-openstackclient/
56
57 - parameter:
58     name: build-tag
59     parameters:
60       - string:
61           name: BUILD_TAG
62           default: ''
63           description: 'Tag in Git to checkout'
64
65 - parameter:
66     name: controller-version-parameter
67     parameters:
68       - string:
69           name: ODL_VERSION
70           default: '{odl_version}'
71           description: 'Controller version (for use with openstacks networking_odl project)'
72
73 - parameter:
74     name: patches-to-build-parameter
75     parameters:
76       - string:
77           name: PATCHES_TO_BUILD
78           default: ''
79           description: |
80             Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47)
81
82 - parameter:
83     name: run-test-parameter
84     parameters:
85       - string:
86           name: RUNTEST
87           default: '{run-test}'
88           description: 'Set true to run test after build'
89
90 - parameter:
91     name: distribution-branch-to-build-parameter
92     parameters:
93       - string:
94           name: DISTRIBUTION_BRANCH_TO_BUILD
95           default: 'master'
96           description: 'distribution repo branch to build with'
97
98 - parameter:
99     name: p2zip-parameter
100     parameters:
101       - string:
102           name: P2ZIP_URL
103           default: ''
104           description: 'Nexus staging profile id'
105
106 - parameter:
107     name: stage-id-parameter
108     parameters:
109       - string:
110           name: STAGING_PROFILE_ID
111           default: '{stage-id}'
112           description: 'Nexus staging profile id'
113
114 - parameter:
115     name: maven-exec
116     parameters:
117       - string:
118           name: MVN
119           default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
120           description: 'Maven selector to be used by shell scripts'
121
122 - scm:
123     name: git-scm
124     scm:
125       - git:
126           credentials-id: 'opendaylight-jenkins-ssh'
127           url: '$GIT_BASE'
128           refspec: ''
129           branches:
130             - 'origin/{branch}'
131           skip-tag: true
132           wipe-workspace: true
133
134 - scm:
135     name: git-scm-with-submodules
136     scm:
137       - git:
138           credentials-id: 'opendaylight-jenkins-ssh'
139           url: '$GIT_BASE'
140           refspec: ''
141           branches:
142             - 'refs/heads/{branch}'
143           skip-tag: true
144           wipe-workspace: true
145           submodule:
146             recursive: true
147
148 - scm:
149     name: gerrit-trigger-scm
150     scm:
151       - git:
152           credentials-id: 'opendaylight-jenkins-ssh'
153           url: '$GIT_BASE'
154           refspec: '{refspec}'
155           branches:
156             - 'origin/$GERRIT_BRANCH'
157           skip-tag: true
158           choosing-strategy: '{choosing-strategy}'
159
160 - wrapper:
161     name: build-timeout
162     wrappers:
163       - timeout:
164           type: absolute
165           timeout: 360
166           fail: true
167
168 # This is a single macro to use for all jobs who vote on every (relevant) patch set.
169 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
170 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
171 # Arguments:
172 #     server: name of gerrit server to listen to
173 #     project: pattern to match triggering projects
174 #     branch: triggering branch name
175 #     files: pattern to match triggering filepaths
176 - trigger:
177     name: gerrit-trigger-patch-submitted
178     triggers:
179       - gerrit:
180           server-name: '{server}'
181           trigger-on:
182             - patchset-created-event:
183                 exclude-drafts: true
184                 exclude-trivial-rebase: false
185                 exclude-no-code-change: false
186             - draft-published-event
187             - comment-added-contains-event:
188                 comment-contains-value: recheck
189           projects:
190             - project-compare-type: ANT
191               project-pattern: '{project}'
192               branches:
193                 - branch-compare-type: ANT
194                   branch-pattern: '**/{branch}'
195               file-paths:
196                 - compare-type: ANT
197                   pattern: '{files}'
198
199 # TODO: Unify argument names across gerrit-trigger-* macros.
200 - trigger:
201     name: gerrit-trigger-patch-merged
202     triggers:
203       - gerrit:
204           server-name: '{server-name}'
205           trigger-on:
206             - change-merged-event
207             - comment-added-contains-event:
208                 comment-contains-value: 'remerge'
209           projects:
210             - project-compare-type: 'ANT'
211               project-pattern: '{name}'
212               branches:
213                 - branch-compare-type: 'ANT'
214                   branch-pattern: '**/{branch}'
215           skip-vote:
216             successful: true
217             failed: true
218             unstable: true
219             notbuilt: true
220           # Force Jenkins always vote the values it should already have voted
221           # during the prior verify phase
222           override-votes: true
223           gerrit-build-started-verified-value: 1
224           gerrit-build-successful-verified-value: 1
225           gerrit-build-failed-verified-value: 1
226           gerrit-build-unstable-verified-value: 1
227           gerrit-build-notbuilt-verified-value: 1
228           gerrit-build-started-codereview-value: 0
229           gerrit-build-successful-codereview-value: 0
230           gerrit-build-failed-codereview-value: 0
231           gerrit-build-unstable-codereview-value: 0
232           gerrit-build-notbuilt-codereview-value: 0
233
234 # TODO: Unify argument names across gerrit-trigger-* macros.
235 - trigger:
236     name: gerrit-trigger-patch-sonar
237     triggers:
238       - gerrit:
239           server-name: '{server-name}'
240           trigger-on:
241             - comment-added-contains-event:
242                 comment-contains-value: 'run-sonar'
243           projects:
244             - project-compare-type: 'ANT'
245               project-pattern: '{name}'
246               branches:
247                 - branch-compare-type: 'ANT'
248                   branch-pattern: '**/master'
249           skip-vote:
250             successful: true
251             failed: true
252             unstable: true
253             notbuilt: true
254
255 - publisher:
256     name: archive-artifacts
257     publishers:
258       - archive:
259           artifacts: '{artifacts}'
260           allow-empty: true
261           fingerprint: true
262           latest-only: true
263
264 - publisher:
265     name: email-notification
266     publishers:
267       - email-ext:
268           recipients: '{email-recipients}'
269           reply-to: ''
270           content-type: default
271           subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
272           body: |
273               $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
274
275               Please refer to the logs server URL for console logs when possible
276               and use the Jenkins Build URL as a last resort.
277
278               Console Logs URL:
279               https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
280
281               Jenkins Build URL:
282               $BUILD_URL
283           unstable: true
284           fixed: true
285           send-to:
286             - recipients
287
288 - publisher:
289     name: jacoco-report
290     publishers:
291       - jacoco:
292           exec-pattern: "**/**.exec"
293           class-pattern: "**/classes"
294           source-pattern: "**/src/main/java"
295           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
296           status-update: true
297           targets:
298             - branch:
299                 healthy: 10
300                 unhealthy: 20
301             - method:
302                 healthy: 50
303                 unhealthy: 40
304
305 - builder:
306     name: opendaylight-infra-stack
307     # opendaylight-infra-stack.sh has a required variable {stack-template} that
308     # must be passed into this macro.
309     builders:
310       - shell: !include-raw: opendaylight-infra-stack.sh
311       - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh
312
313 - builder:
314     name: opendaylight-infra-push-gerrit-patch
315     # opendaylight-infra-push-gerrit-patch.sh allows a job to push a patch to
316     # Gerrit in an automated fashion. This is meant for tasks that creates
317     # the same patch regularly and needs the ability to detect if an unreviewed
318     # patch already exists. In which case it will update the existing patch.
319     #
320     # Note: This patch assumes the $WORKSPACE contains the project repo with
321     #       the files changed already "git add" and waiting for a "git commit" call.
322     #
323     # This script requires the following JJB variables to be passed in:
324     #     {project}       Gerrit project-name
325     #     {gerrit-topic}  Gerrit topic, please make a unique topic.
326     #     {gerrit-commit-message} Commit message to assign to commit.
327     # NOTE: Requires git review to be installed on node.
328     builders:
329       - shell: !include-raw: opendaylight-infra-push-gerrit-patch.sh
330
331 - builder:
332     name: distribute-build-url
333     # Place URL of the current run of a build job to a file at given path.
334     builders:
335       - shell: |
336           #!/bin/bash
337           set +e  # DO NOT fail script if command returns non-zero.
338
339           echo "$BUILD_URL" > {path}/build.url
340
341           # DO NOT fail the build if the echo failed.
342           exit 0
343
344 - builder:
345     name: wipe-org-opendaylight-repo
346     builders:
347       - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi'
348
349 - builder:
350     name: wipe-local-maven-repo
351     builders:
352       - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi'
353
354 - builder:
355     name: jacoco-nojava-workaround
356     builders:
357       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
358
359 - builder:
360     name: check-clm
361     builders:
362       - sonatype-clm:
363           application-name: '{application-name}'
364
365 - builder:
366     name: releng-check-unicode
367     builders:
368       - shell: |
369           $WORKSPACE/scripts/check-unicode.sh jjb/
370
371 - builder:
372     name: provide-maven-settings
373     builders:
374       - config-file-provider:
375           files:
376             - file-id: '{global-settings-file}'
377               variable: 'GLOBAL_SETTINGS_FILE'
378             - file-id: '{settings-file}'
379               variable: 'SETTINGS_FILE'
380
381 - builder:
382     name: releng-fetch-p2zip-if-necessary
383     builders:
384       - shell: |
385           # Cleanup any existing zips and metadata before we download the new update site
386           rm -f *.zip *.xml
387       - conditional-step:
388           condition-kind: strings-match
389           condition-string1: '$P2ZIP_URL'
390           condition-string2: ''
391           condition-basedir: workspace
392           steps:
393             # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT
394             - maven-target:
395                 maven-version: '{maven-version}'
396                 goals: >
397                     org.apache.maven.plugins:maven-dependency-plugin:get
398                     org.apache.maven.plugins:maven-dependency-plugin:copy
399                     -V -B
400                     -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip
401                     -DoutputDirectory=$WORKSPACE
402                 settings: '{settings}'
403                 global-settings: '{global-settings}'
404
405 - builder:
406     name: releng-generate-p2pom
407     builders:
408       - shell: !include-raw: include-raw-generate-p2pom.sh
409
410 - builder:
411     name: releng-update-p2composite-metadata
412     builders:
413       - shell: !include-raw: include-raw-update-p2composite-metadata.sh
414       - conditional-step:
415           condition-kind: file-exists
416           condition-filename: deploy-composite-repo.xml
417           condition-basedir: workspace
418           steps:
419             - maven-target:
420                 maven-version: '{maven-version}'
421                 pom: 'deploy-composite-repo.xml'
422                 goals: 'clean deploy -V -B -Dmaven.repo.local=/tmp/r'
423                 settings: '{settings}'
424                 global-settings: '{global-settings}'
425
426 - builder:
427     name: releng-stage-release
428     builders:
429       - shell: !include-raw: include-raw-stage-release.sh
430
431 - wrapper:
432     # This wrapper is required for all jobs as it configures the wrappers
433     # needed by OpenDaylight infra.
434     name: opendaylight-infra-wrappers
435     wrappers:
436       - mask-passwords
437       - config-file-provider:
438           files:
439             - file-id: npmrc
440               target: '$HOME/.npmrc'
441             - file-id: rackspace-heat
442               target: '$HOME/.config/openstack/clouds.yaml'
443       - timeout:
444           type: absolute
445           timeout: '{build-timeout}'
446           timeout-var: 'BUILD_TIMEOUT'
447           fail: true
448       - timestamps
449       - ssh-agent-credentials:
450           users:
451             - 'opendaylight-jenkins-ssh'
452       - openstack:
453           single-use: true
454
455 - builder:
456     name: packer-validate
457     builders:
458       - config-file-provider:
459           files:
460             - file-id: 'packer-cloud-env'
461               variable: 'CLOUDENV'
462       - shell: |
463           #!/bin/bash
464           # Ensure we fail the job if any steps fail.
465           set -eu -o pipefail
466           cd packer
467           varfiles="../packer/vars/*"
468           templates="../packer/templates/*"
469           provision="../packer/provision/*.sh"
470           for v in $varfiles; do
471               [[ "${v##*/}" =~ ^(cloud-env.*)$ ]] && continue
472               for t in $templates; do
473                   export PACKER_LOG="yes" && \
474                   export PACKER_LOG_PATH="packer-validate-${v##*/}-${t##*/}.log" && \
475                               packer.io validate -var-file=$CLOUDENV \
476                               -var-file=$v $t
477                   if [ $? -ne 0 ]; then
478                       break
479                   fi
480               done
481           done
482
483 - builder:
484     name: packer-build
485     builders:
486       - config-file-provider:
487           files:
488             - file-id: 'packer-cloud-env'
489               variable: 'CLOUDENV'
490       - shell: |
491           #!/bin/bash
492           # Ensure we fail the job if any steps fail.
493           set -eu -o pipefail
494           cd packer
495           export PACKER_LOG="yes" && \
496           export PACKER_LOG_PATH="packer-build.log" && \
497                       packer.io build -color=false -var-file=$CLOUDENV \
498                        -var-file=../packer/vars/{platform}.json \
499                        ../packer/templates/{template}.json
500           # Split public and private cloud logs
501           grep -e 'public_cloud' packer-build.log > packer-build_public_cloud.log  2>&1
502           grep -e 'private_cloud' packer-build.log > packer-build_private_cloud.log 2>&1
503
504 - builder:
505     # TODO: Verify signature after downloading users public key from a locally created
506     # repository instead of the public keymesh. This requires a process in place to get ODL
507     # developers public keys into a local repository without increasing the job thoughput.
508     name: verify-gpg-signature
509     builders:
510       - shell: !include-raw: include-raw-verify-gpg-signatures.sh
511
512 - builder:
513     name: opendaylight-infra-jjbini
514     builders:
515       - config-file-provider:
516           files:
517             - file-id: 'jjbini'
518               target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini'
519
520 - builder:
521     name: distribution-check-wipe
522     # Step zero: Wipe file repositories up front.
523     builders:
524       - shell: |
525           echo "wipe r: the local Maven repository"
526           rm -rfv /tmp/r
527           echo "wipe n: the fake remote (Nexus) repository"
528           rm -rfv /tmp/n
529           echo "wipe t: the transient repository used in some manipulations"
530           rm -rfv /tmp/t
531
532 - builder:
533     name: distribution-check-build-project
534     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
535     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
536     # Arguments:
537     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
538     builders:
539       - maven-target:
540           maven-version: 'mvn33'
541           pom: '{pom}'
542           goals: |
543               clean deploy dependency:tree
544               -DoutputFile=dependency_tree.txt
545               -Pq
546               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
547               {mvn-opts}
548           java-opts:
549             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
550           settings: 'integration-settings'
551           settings-type: cfp
552           global-settings: 'global-settings'
553           global-settings-type: cfp
554
555 - builder:
556     name: distribution-check-verify-groupid
557     # Step two: Verify all deployed artifacts belong to the project's groupId.
558     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
559     # The correct directory is derived from $GERRIT_PROJECT.
560     # TODO: Verify all deployed artifacts are snapshots.
561     # Arguments:
562     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
563     builders:
564       - shell: |
565           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
566           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
567           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
568           rm -rf /tmp/n
569           mv /tmp/t /tmp/n
570
571 - builder:
572     name: distribution-check-download-deps
573     # Step three: Online build of integration distribution.
574     # This step is mainly used for downloading other project artifacts.
575     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
576     # Arguments:
577     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
578     builders:
579       - maven-target:
580           maven-version: 'mvn33'
581           pom: '{dist-pom}'
582           goals: |
583               clean install dependency:tree
584               -DoutputFile=dependency_tree.txt
585               -Pq
586               {mvn-opts}
587           java-opts:
588             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
589           settings: 'integration-settings'
590           settings-type: cfp
591           global-settings: 'global-settings'
592           global-settings-type: cfp
593
594 - builder:
595     name: distribution-check-delete-snapshots
596     # Step four: Delete snapshot artifacts from the local repository.
597     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
598     # Also other files related to maven repository resolution are removed,
599     # and then empty directories are removed, in order to simplify debugging.
600     builders:
601       - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh
602
603 - builder:
604     name: distribution-check-configure-remotes
605     # Now the ugly part. It seems that the only way to tell Maven 2+
606     # which remote repositories to use is via settings.xml file.
607     # So we create such a file here, but it needs most of odlparent:settings.xml
608     builders:
609       - shell: |
610           echo '
611           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
612             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
613             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
614             https://maven.apache.org/xsd/settings-1.0.0.xsd">
615             <profiles>
616               <profile>
617                 <id>opendaylight-release</id>
618                 <repositories>
619                   <repository>
620                     <id>opendaylight-mirror</id>
621                     <name>opendaylight</name>
622                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
623                     <releases><updatePolicy>never</updatePolicy></releases>
624                     <snapshots><enabled>false</enabled></snapshots>
625                   </repository>
626                 </repositories>
627                 <pluginRepositories>
628                   <pluginRepository>
629                     <id>opendaylight-plugin-mirror</id>
630                     <name>opendaylight-plugin</name>
631                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
632                     <releases><updatePolicy>never</updatePolicy></releases>
633                     <snapshots><enabled>false</enabled></snapshots>
634                   </pluginRepository>
635                 </pluginRepositories>
636               </profile>
637               <profile>
638                 <id>file-snapshots</id>
639                 <repositories>
640                   <repository>
641                     <id>file-snapshots</id>
642                     <name>file</name>
643                     <url>file:///tmp/n/</url>
644                     <releases><enabled>false</enabled></releases>
645                   </repository>
646                 </repositories>
647                 <pluginRepositories>
648                   <pluginRepository>
649                     <id>file-plugin-snapshots</id>
650                     <name>file-plugin</name>
651                     <url>file:///tmp/n/</url>
652                     <releases><enabled>false</enabled></releases>
653                   </pluginRepository>
654                 </pluginRepositories>
655               </profile>
656             </profiles>
657             <activeProfiles>
658               <activeProfile>file-snapshots</activeProfile>
659               <activeProfile>opendaylight-release</activeProfile>
660             </activeProfiles>
661           </settings>
662           ' > fake_remotes.xml
663           # Notes: The settings are minimal in order to detect breakage scenarios,
664           # while allowing for the following quirks:
665           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
666           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
667           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
668
669 - builder:
670     name: distribution-check-repeat-build
671     # Step five: Repeat the distribution build but with the new settings.
672     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
673     # which faithfully reproduces conditions in later verify-like job runs.
674     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode"
675     # This is where SingleFeatureTest is not skipped.
676     # Arguments:
677     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
678     builders:
679       - maven-target:
680           maven-version: 'mvn33'
681           pom: '{dist-pom}'
682           goals: |
683               clean install dependency:tree
684               -DoutputFile=dependency_tree.txt -s fake_remotes.xml
685               -Pq
686               -DskipTests=false
687               {mvn-opts}
688           java-opts:
689             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'