da35e0169e217c6ead795b173dfaa97c7ab50bb8
[releng/builder.git] / jjb / releng-macros.yaml
1 # OLD Releng macros
2 ---
3 - property:
4     name: opendaylight-infra-properties
5     properties:
6       - build-discarder:
7           days-to-keep: '{build-days-to-keep}'
8           num-to-keep: 40
9           # Need to keep artifacts for at least 1 day as some projects need to
10           # be able to validate their artifacts and only allowing limited
11           # number of artifacts could make fast moving jobs lost their
12           # artifacts before the developers can test it.
13           artifact-days-to-keep: 1
14
15 - parameter:
16     name: opendaylight-infra-parameters
17     parameters:
18       - string:
19           name: PROJECT
20           default: '{project}'
21           description: 'Parameter to identify an ODL Gerrit project'
22       - string:
23           name: ARCHIVE_ARTIFACTS
24           default: >
25               {artifacts}
26               # Before adding more here, beware of https://jira.linuxfoundation.org/browse/RELENG-280
27               **/target/surefire-reports/*-output.txt
28               **/target/failsafe-reports/failsafe-summary.xml
29               **/hs_err_*.log
30               **/target/feature/feature.xml
31               **/*.hprof
32           description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
33       - string:
34           name: GERRIT_PROJECT
35           default: '{project}'
36           description: "GERRIT_PROJECT parameter if not given by trigger"
37       - string:
38           name: GERRIT_BRANCH
39           default: '{branch}'
40           description: "JJB configured GERRIT_BRANCH parameter"
41       - string:
42           name: GERRIT_REFSPEC
43           default: '{refspec}'
44           description: "GERRIT_REFSPEC parameter if not given by trigger"
45       - string:
46           name: STACK_NAME
47           default: '$SILO-$JOB_NAME-$BUILD_NUMBER'
48           description: "Used by Heat to generate a unique stack & vm name"
49       - string:
50           name: OS_CLOUD
51           default: '{os-cloud}'
52           description: |
53               The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
54               variable name that is significant to openstack client as a
55               environment variable. Please refer to the documentation for
56               further details.
57               https://docs.openstack.org/developer/python-openstackclient/
58
59 - parameter:
60     name: build-tag
61     parameters:
62       - string:
63           name: BUILD_TAG
64           default: ''
65           description: 'Tag in Git to checkout'
66
67 - parameter:
68     name: controller-version-parameter
69     parameters:
70       - string:
71           name: ODL_VERSION
72           default: '{odl_version}'
73           description: 'Controller version (for use with openstacks networking_odl project)'
74
75 - parameter:
76     name: patches-to-build-parameter
77     parameters:
78       - string:
79           name: PATCHES_TO_BUILD
80           default: ''
81           description: |
82             Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47)
83
84 - parameter:
85     name: run-test-parameter
86     parameters:
87       - string:
88           name: RUNTEST
89           default: '{run-test}'
90           description: 'Set true to run test after build'
91
92 - parameter:
93     name: distribution-branch-to-build-parameter
94     parameters:
95       - string:
96           name: DISTRIBUTION_BRANCH_TO_BUILD
97           default: 'master'
98           description: 'distribution repo branch to build with'
99
100 - parameter:
101     name: p2zip-parameter
102     parameters:
103       - string:
104           name: P2ZIP_URL
105           default: ''
106           description: 'Nexus staging profile id'
107
108 - parameter:
109     name: stage-id-parameter
110     parameters:
111       - string:
112           name: STAGING_PROFILE_ID
113           default: '{stage-id}'
114           description: 'Nexus staging profile id'
115
116 - parameter:
117     name: maven-exec
118     parameters:
119       - string:
120           name: MVN
121           default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
122           description: 'Maven selector to be used by shell scripts'
123
124 - scm:
125     name: git-scm
126     scm:
127       - git:
128           credentials-id: 'opendaylight-jenkins-ssh'
129           url: '$GIT_BASE'
130           refspec: ''
131           branches:
132             - 'origin/{branch}'
133           skip-tag: true
134           wipe-workspace: true
135
136 - scm:
137     name: git-scm-with-submodules
138     scm:
139       - git:
140           credentials-id: 'opendaylight-jenkins-ssh'
141           url: '$GIT_BASE'
142           refspec: ''
143           branches:
144             - 'refs/heads/{branch}'
145           skip-tag: true
146           wipe-workspace: true
147           submodule:
148             recursive: true
149
150 - scm:
151     name: gerrit-trigger-scm
152     scm:
153       - git:
154           credentials-id: 'opendaylight-jenkins-ssh'
155           url: '$GIT_BASE'
156           refspec: '{refspec}'
157           branches:
158             - 'origin/$GERRIT_BRANCH'
159           skip-tag: true
160           choosing-strategy: '{choosing-strategy}'
161
162 - wrapper:
163     name: build-timeout
164     wrappers:
165       - timeout:
166           type: absolute
167           timeout: 360
168           fail: true
169
170 # This is a single macro to use for all jobs who vote on every (relevant) patch set.
171 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
172 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
173 # Arguments:
174 #     server: name of gerrit server to listen to
175 #     project: pattern to match triggering projects
176 #     branch: triggering branch name
177 #     files: pattern to match triggering filepaths
178 - trigger:
179     name: gerrit-trigger-patch-submitted
180     triggers:
181       - gerrit:
182           server-name: '{server}'
183           trigger-on:
184             - patchset-created-event:
185                 exclude-drafts: true
186                 exclude-trivial-rebase: false
187                 exclude-no-code-change: false
188             - draft-published-event
189             - comment-added-contains-event:
190                 comment-contains-value: recheck
191           projects:
192             - project-compare-type: ANT
193               project-pattern: '{project}'
194               branches:
195                 - branch-compare-type: ANT
196                   branch-pattern: '**/{branch}'
197               file-paths:
198                 - compare-type: ANT
199                   pattern: '{files}'
200
201 # TODO: Unify argument names across gerrit-trigger-* macros.
202 - trigger:
203     name: gerrit-trigger-patch-merged
204     triggers:
205       - gerrit:
206           server-name: '{server-name}'
207           trigger-on:
208             - change-merged-event
209             - comment-added-contains-event:
210                 comment-contains-value: 'remerge'
211           projects:
212             - project-compare-type: 'ANT'
213               project-pattern: '{name}'
214               branches:
215                 - branch-compare-type: 'ANT'
216                   branch-pattern: '**/{branch}'
217           skip-vote:
218             successful: true
219             failed: true
220             unstable: true
221             notbuilt: true
222           # Force Jenkins always vote the values it should already have voted
223           # during the prior verify phase
224           override-votes: true
225           gerrit-build-started-verified-value: 1
226           gerrit-build-successful-verified-value: 1
227           gerrit-build-failed-verified-value: 1
228           gerrit-build-unstable-verified-value: 1
229           gerrit-build-notbuilt-verified-value: 1
230           gerrit-build-started-codereview-value: 0
231           gerrit-build-successful-codereview-value: 0
232           gerrit-build-failed-codereview-value: 0
233           gerrit-build-unstable-codereview-value: 0
234           gerrit-build-notbuilt-codereview-value: 0
235
236 # TODO: Unify argument names across gerrit-trigger-* macros.
237 - trigger:
238     name: gerrit-trigger-patch-sonar
239     triggers:
240       - gerrit:
241           server-name: '{server-name}'
242           trigger-on:
243             - comment-added-contains-event:
244                 comment-contains-value: 'run-sonar'
245           projects:
246             - project-compare-type: 'ANT'
247               project-pattern: '{name}'
248               branches:
249                 - branch-compare-type: 'ANT'
250                   branch-pattern: '**/master'
251           skip-vote:
252             successful: true
253             failed: true
254             unstable: true
255             notbuilt: true
256
257 - publisher:
258     name: email-notification
259     publishers:
260       - email-ext:
261           recipients: '{email-recipients}'
262           reply-to: ''
263           content-type: default
264           subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
265           body: |
266               $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
267
268               Please refer to the logs server URL for console logs when possible
269               and use the Jenkins Build URL as a last resort.
270
271               Console Logs URL:
272               https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
273
274               Jenkins Build URL:
275               $BUILD_URL
276           unstable: true
277           fixed: true
278           send-to:
279             - recipients
280
281 - publisher:
282     name: jacoco-report
283     publishers:
284       - jacoco:
285           exec-pattern: "**/**.exec"
286           class-pattern: "**/classes"
287           source-pattern: "**/src/main/java"
288           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
289           status-update: true
290           targets:
291             - branch:
292                 healthy: 10
293                 unhealthy: 20
294             - method:
295                 healthy: 50
296                 unhealthy: 40
297
298 - builder:
299     # Deploys a maven site to Nexus using lftools nexus-zip command
300     name: opendaylight-infra-deploy-maven-site
301     builders:
302       - lf-provide-maven-settings:
303           global-settings-file: global-settings
304           settings-file: '{settings-file}'
305       - lf-infra-create-netrc:
306           server-id: opendaylight-site
307       - shell: !include-raw-escape:
308           - global-jjb/shell/lftools-install.sh
309           - opendaylight-infra-deploy-maven-site.sh
310       - lf-provide-maven-settings-cleanup
311
312 - builder:
313     name: opendaylight-infra-stack
314     # opendaylight-infra-stack.sh has a required variable {stack-template} that
315     # must be passed into this macro.
316     builders:
317       - shell: !include-raw: opendaylight-infra-stack.sh
318       - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh
319
320 - builder:
321     name: opendaylight-infra-push-gerrit-patch
322     # opendaylight-infra-push-gerrit-patch.sh allows a job to push a patch to
323     # Gerrit in an automated fashion. This is meant for tasks that creates
324     # the same patch regularly and needs the ability to detect if an unreviewed
325     # patch already exists. In which case it will update the existing patch.
326     #
327     # Note: This patch assumes the $WORKSPACE contains the project repo with
328     #       the files changed already "git add" and waiting for a "git commit" call.
329     #
330     # This script requires the following JJB variables to be passed in:
331     #     {project}       Gerrit project-name
332     #     {gerrit-topic}  Gerrit topic, please make a unique topic.
333     #     {gerrit-commit-message} Commit message to assign to commit.
334     # NOTE: Requires git review to be installed on node.
335     builders:
336       - shell: !include-raw: opendaylight-infra-push-gerrit-patch.sh
337
338 - builder:
339     name: distribute-build-url
340     # Place URL of the current run of a build job to a file at given path.
341     builders:
342       - shell: |
343           #!/bin/bash
344           set +e  # DO NOT fail script if command returns non-zero.
345
346           echo "$BUILD_URL" > {path}/build.url
347
348           # DO NOT fail the build if the echo failed.
349           exit 0
350
351 - builder:
352     name: wipe-org-opendaylight-repo
353     builders:
354       - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi'
355
356 - builder:
357     name: wipe-local-maven-repo
358     builders:
359       - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi'
360
361 - builder:
362     name: jacoco-nojava-workaround
363     builders:
364       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
365
366 - builder:
367     name: check-clm
368     builders:
369       - sonatype-clm:
370           application-name: '{application-name}'
371
372 - builder:
373     name: releng-check-unicode
374     builders:
375       - shell: |
376           $WORKSPACE/scripts/check-unicode.sh jjb/
377
378 - builder:
379     name: provide-maven-settings
380     builders:
381       - config-file-provider:
382           files:
383             - file-id: '{global-settings-file}'
384               variable: 'GLOBAL_SETTINGS_FILE'
385             - file-id: '{settings-file}'
386               variable: 'SETTINGS_FILE'
387
388 - builder:
389     name: releng-fetch-p2zip-if-necessary
390     builders:
391       - shell: |
392           # Cleanup any existing zips and metadata before we download the new update site
393           rm -f *.zip *.xml
394       - conditional-step:
395           condition-kind: strings-match
396           condition-string1: '$P2ZIP_URL'
397           condition-string2: ''
398           condition-basedir: workspace
399           steps:
400             # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT
401             - maven-target:
402                 maven-version: '{maven-version}'
403                 goals: >
404                     org.apache.maven.plugins:maven-dependency-plugin:get
405                     org.apache.maven.plugins:maven-dependency-plugin:copy
406                     -V -B
407                     -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip
408                     -DoutputDirectory=$WORKSPACE
409                 settings: '{settings}'
410                 global-settings: '{global-settings}'
411
412 - builder:
413     name: releng-generate-p2pom
414     builders:
415       - shell: !include-raw: include-raw-generate-p2pom.sh
416
417 - builder:
418     name: releng-update-p2composite-metadata
419     builders:
420       - shell: !include-raw: include-raw-update-p2composite-metadata.sh
421       - conditional-step:
422           condition-kind: file-exists
423           condition-filename: deploy-composite-repo.xml
424           condition-basedir: workspace
425           steps:
426             - maven-target:
427                 maven-version: '{maven-version}'
428                 pom: 'deploy-composite-repo.xml'
429                 goals: 'clean deploy -V -B -Dmaven.repo.local=/tmp/r'
430                 settings: '{settings}'
431                 global-settings: '{global-settings}'
432
433 - builder:
434     name: releng-stage-release
435     builders:
436       - shell: !include-raw: include-raw-stage-release.sh
437
438 - wrapper:
439     # This wrapper is required for all jobs as it configures the wrappers
440     # needed by OpenDaylight infra.
441     name: opendaylight-infra-wrappers
442     wrappers:
443       - mask-passwords
444       - config-file-provider:
445           files:
446             - file-id: npmrc
447               target: '$HOME/.npmrc'
448             - file-id: rackspace-heat
449               target: '$HOME/.config/openstack/clouds.yaml'
450       - timeout:
451           type: absolute
452           timeout: '{build-timeout}'
453           timeout-var: 'BUILD_TIMEOUT'
454           fail: true
455       - timestamps
456       - ssh-agent-credentials:
457           users:
458             - 'opendaylight-jenkins-ssh'
459       - openstack:
460           single-use: true
461
462 - builder:
463     name: packer-validate
464     builders:
465       - config-file-provider:
466           files:
467             - file-id: 'packer-cloud-env'
468               variable: 'CLOUDENV'
469       - shell: |
470           #!/bin/bash
471           # Ensure we fail the job if any steps fail.
472           set -eu -o pipefail
473           cd packer
474           varfiles="../packer/vars/*"
475           templates="../packer/templates/*"
476           provision="../packer/provision/*.sh"
477           for v in $varfiles; do
478               [[ "${v##*/}" =~ ^(cloud-env.*)$ ]] && continue
479               for t in $templates; do
480                   export PACKER_LOG="yes" && \
481                   export PACKER_LOG_PATH="packer-validate-${v##*/}-${t##*/}.log" && \
482                               packer.io validate -var-file=$CLOUDENV \
483                               -var-file=$v $t
484                   if [ $? -ne 0 ]; then
485                       break
486                   fi
487               done
488           done
489
490 - builder:
491     name: packer-build
492     builders:
493       - config-file-provider:
494           files:
495             - file-id: 'packer-cloud-env'
496               variable: 'CLOUDENV'
497       - shell: |
498           #!/bin/bash
499           # Ensure we fail the job if any steps fail.
500           set -eu -o pipefail
501           cd packer
502           export PACKER_LOG="yes" && \
503           export PACKER_LOG_PATH="packer-build.log" && \
504                       packer.io build -color=false -var-file=$CLOUDENV \
505                        -var-file=../packer/vars/{platform}.json \
506                        ../packer/templates/{template}.json
507           # Split public and private cloud logs
508           grep -e 'public_cloud' packer-build.log > packer-build_public_cloud.log  2>&1
509           grep -e 'private_cloud' packer-build.log > packer-build_private_cloud.log 2>&1
510
511 - builder:
512     # TODO: Verify signature after downloading users public key from a locally created
513     # repository instead of the public keymesh. This requires a process in place to get ODL
514     # developers public keys into a local repository without increasing the job thoughput.
515     name: verify-gpg-signature
516     builders:
517       - shell: !include-raw: include-raw-verify-gpg-signatures.sh
518
519 - builder:
520     name: opendaylight-infra-jjbini
521     builders:
522       - config-file-provider:
523           files:
524             - file-id: 'jjbini'
525               target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini'
526
527 - builder:
528     name: distribution-check-wipe
529     # Step zero: Wipe file repositories up front.
530     builders:
531       - shell: |
532           echo "wipe r: the local Maven repository"
533           rm -rfv /tmp/r
534           echo "wipe n: the fake remote (Nexus) repository"
535           rm -rfv /tmp/n
536           echo "wipe t: the transient repository used in some manipulations"
537           rm -rfv /tmp/t
538
539 - builder:
540     name: distribution-check-build-project
541     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
542     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
543     # Arguments:
544     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
545     builders:
546       - maven-target:
547           maven-version: 'mvn33'
548           pom: '{pom}'
549           goals: |
550               clean deploy dependency:tree
551               -DoutputFile=dependency_tree.txt
552               -Pq
553               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
554               {mvn-opts}
555           java-opts:
556             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
557           settings: 'integration-settings'
558           settings-type: cfp
559           global-settings: 'global-settings'
560           global-settings-type: cfp
561
562 - builder:
563     name: distribution-check-verify-groupid
564     # Step two: Verify all deployed artifacts belong to the project's groupId.
565     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
566     # The correct directory is derived from $GERRIT_PROJECT.
567     # TODO: Verify all deployed artifacts are snapshots.
568     # Arguments:
569     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
570     builders:
571       - shell: |
572           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
573           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
574           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
575           rm -rf /tmp/n
576           mv /tmp/t /tmp/n
577
578 - builder:
579     name: distribution-check-download-deps
580     # Step three: Online build of integration distribution.
581     # This step is mainly used for downloading other project artifacts.
582     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
583     # Arguments:
584     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
585     builders:
586       - maven-target:
587           maven-version: 'mvn33'
588           pom: '{dist-pom}'
589           goals: |
590               clean install dependency:tree
591               -DoutputFile=dependency_tree.txt
592               -Pq
593               {mvn-opts}
594           java-opts:
595             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
596           settings: 'integration-settings'
597           settings-type: cfp
598           global-settings: 'global-settings'
599           global-settings-type: cfp
600
601 - builder:
602     name: distribution-check-delete-snapshots
603     # Step four: Delete snapshot artifacts from the local repository.
604     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
605     # Also other files related to maven repository resolution are removed,
606     # and then empty directories are removed, in order to simplify debugging.
607     builders:
608       - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh
609
610 - builder:
611     name: distribution-check-configure-remotes
612     # Now the ugly part. It seems that the only way to tell Maven 2+
613     # which remote repositories to use is via settings.xml file.
614     # So we create such a file here, but it needs most of odlparent:settings.xml
615     builders:
616       - shell: |
617           echo '
618           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
619             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
620             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
621             https://maven.apache.org/xsd/settings-1.0.0.xsd">
622             <profiles>
623               <profile>
624                 <id>opendaylight-release</id>
625                 <repositories>
626                   <repository>
627                     <id>opendaylight-mirror</id>
628                     <name>opendaylight</name>
629                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
630                     <releases><updatePolicy>never</updatePolicy></releases>
631                     <snapshots><enabled>false</enabled></snapshots>
632                   </repository>
633                 </repositories>
634                 <pluginRepositories>
635                   <pluginRepository>
636                     <id>opendaylight-plugin-mirror</id>
637                     <name>opendaylight-plugin</name>
638                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
639                     <releases><updatePolicy>never</updatePolicy></releases>
640                     <snapshots><enabled>false</enabled></snapshots>
641                   </pluginRepository>
642                 </pluginRepositories>
643               </profile>
644               <profile>
645                 <id>file-snapshots</id>
646                 <repositories>
647                   <repository>
648                     <id>file-snapshots</id>
649                     <name>file</name>
650                     <url>file:///tmp/n/</url>
651                     <releases><enabled>false</enabled></releases>
652                   </repository>
653                 </repositories>
654                 <pluginRepositories>
655                   <pluginRepository>
656                     <id>file-plugin-snapshots</id>
657                     <name>file-plugin</name>
658                     <url>file:///tmp/n/</url>
659                     <releases><enabled>false</enabled></releases>
660                   </pluginRepository>
661                 </pluginRepositories>
662               </profile>
663             </profiles>
664             <activeProfiles>
665               <activeProfile>file-snapshots</activeProfile>
666               <activeProfile>opendaylight-release</activeProfile>
667             </activeProfiles>
668           </settings>
669           ' > fake_remotes.xml
670           # Notes: The settings are minimal in order to detect breakage scenarios,
671           # while allowing for the following quirks:
672           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
673           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
674           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
675
676 - builder:
677     name: distribution-check-repeat-build
678     # Step five: Repeat the distribution build but with the new settings.
679     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
680     # which faithfully reproduces conditions in later verify-like job runs.
681     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode"
682     # This is where SingleFeatureTest is not skipped.
683     # Arguments:
684     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
685     builders:
686       - maven-target:
687           maven-version: 'mvn33'
688           pom: '{dist-pom}'
689           goals: |
690               clean install dependency:tree
691               -DoutputFile=dependency_tree.txt -s fake_remotes.xml
692               -Pq
693               -DskipTests=false
694               {mvn-opts}
695           java-opts:
696             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'