Merge "Exclude test cases labeled with skip_if_{stream}"
[releng/builder.git] / jjb / releng-macros.yaml
1 # OLD Releng macros
2 ---
3 - property:
4     name: opendaylight-infra-properties
5     properties:
6       - build-discarder:
7           days-to-keep: '{build-days-to-keep}'
8           num-to-keep: 40
9           # Need to keep artifacts for at least 1 day as some projects need to
10           # be able to validate their artifacts and only allowing limited
11           # number of artifacts could make fast moving jobs lost their
12           # artifacts before the developers can test it.
13           artifact-days-to-keep: 1
14
15 - parameter:
16     name: opendaylight-infra-parameters
17     parameters:
18       - string:
19           name: PROJECT
20           default: '{project}'
21           description: 'Parameter to identify an ODL Gerrit project'
22       - string:
23           name: ARCHIVE_ARTIFACTS
24           default: >
25               {artifacts}
26               **/target/surefire-reports/*.txt
27               **/target/surefire-reports/*.xml
28               **/target/failsafe-reports/*.txt
29               **/target/failsafe-reports/*.xml
30               **/hs_err_*.log
31               **/target/feature/feature.xml
32               **/*.hprof
33           description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
34       - string:
35           name: GERRIT_PROJECT
36           default: '{project}'
37           description: "GERRIT_PROJECT parameter if not given by trigger"
38       - string:
39           name: GERRIT_BRANCH
40           default: '{branch}'
41           description: "JJB configured GERRIT_BRANCH parameter"
42       - string:
43           name: GERRIT_REFSPEC
44           default: '{refspec}'
45           description: "GERRIT_REFSPEC parameter if not given by trigger"
46       - string:
47           name: STACK_NAME
48           default: '$SILO-$JOB_NAME-$BUILD_NUMBER'
49           description: "Used by Heat to generate a unique stack & vm name"
50       - string:
51           name: OS_CLOUD
52           default: '{os-cloud}'
53           description: |
54               The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
55               variable name that is significant to openstack client as a
56               environment variable. Please refer to the documentation for
57               further details.
58               https://docs.openstack.org/developer/python-openstackclient/
59
60 - parameter:
61     name: build-tag
62     parameters:
63       - string:
64           name: BUILD_TAG
65           default: ''
66           description: 'Tag in Git to checkout'
67
68 - parameter:
69     name: controller-version-parameter
70     parameters:
71       - string:
72           name: ODL_VERSION
73           default: '{odl_version}'
74           description: 'Controller version (for use with openstacks networking_odl project)'
75
76 - parameter:
77     name: patches-to-build-parameter
78     parameters:
79       - string:
80           name: PATCHES_TO_BUILD
81           default: ''
82           description: |
83             Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47)
84
85 - parameter:
86     name: run-test-parameter
87     parameters:
88       - string:
89           name: RUNTEST
90           default: '{run-test}'
91           description: 'Set true to run test after build'
92
93 - parameter:
94     name: distribution-branch-to-build-parameter
95     parameters:
96       - string:
97           name: DISTRIBUTION_BRANCH_TO_BUILD
98           default: 'master'
99           description: 'distribution repo branch to build with'
100
101 - parameter:
102     name: p2zip-parameter
103     parameters:
104       - string:
105           name: P2ZIP_URL
106           default: ''
107           description: 'Nexus staging profile id'
108
109 - parameter:
110     name: stage-id-parameter
111     parameters:
112       - string:
113           name: STAGING_PROFILE_ID
114           default: '{stage-id}'
115           description: 'Nexus staging profile id'
116
117 - parameter:
118     name: maven-exec
119     parameters:
120       - string:
121           name: MVN
122           default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
123           description: 'Maven selector to be used by shell scripts'
124
125 - scm:
126     name: git-scm
127     scm:
128       - git:
129           credentials-id: 'opendaylight-jenkins-ssh'
130           url: '$GIT_BASE'
131           refspec: ''
132           branches:
133             - 'origin/{branch}'
134           skip-tag: true
135           wipe-workspace: true
136
137 - scm:
138     name: git-scm-with-submodules
139     scm:
140       - git:
141           credentials-id: 'opendaylight-jenkins-ssh'
142           url: '$GIT_BASE'
143           refspec: ''
144           branches:
145             - 'refs/heads/{branch}'
146           skip-tag: true
147           wipe-workspace: true
148           submodule:
149             recursive: true
150
151 - scm:
152     name: gerrit-trigger-scm
153     scm:
154       - git:
155           credentials-id: 'opendaylight-jenkins-ssh'
156           url: '$GIT_BASE'
157           refspec: '{refspec}'
158           branches:
159             - 'origin/$GERRIT_BRANCH'
160           skip-tag: true
161           choosing-strategy: '{choosing-strategy}'
162
163 - wrapper:
164     name: build-timeout
165     wrappers:
166       - timeout:
167           type: absolute
168           timeout: 360
169           fail: true
170
171 # This is a single macro to use for all jobs who vote on every (relevant) patch set.
172 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
173 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
174 # Arguments:
175 #     server: name of gerrit server to listen to
176 #     project: pattern to match triggering projects
177 #     branch: triggering branch name
178 #     files: pattern to match triggering filepaths
179 - trigger:
180     name: gerrit-trigger-patch-submitted
181     triggers:
182       - gerrit:
183           server-name: '{server}'
184           trigger-on:
185             - patchset-created-event:
186                 exclude-drafts: true
187                 exclude-trivial-rebase: false
188                 exclude-no-code-change: false
189             - draft-published-event
190             - comment-added-contains-event:
191                 comment-contains-value: recheck
192           projects:
193             - project-compare-type: ANT
194               project-pattern: '{project}'
195               branches:
196                 - branch-compare-type: ANT
197                   branch-pattern: '**/{branch}'
198               file-paths:
199                 - compare-type: ANT
200                   pattern: '{files}'
201
202 # TODO: Unify argument names across gerrit-trigger-* macros.
203 - trigger:
204     name: gerrit-trigger-patch-merged
205     triggers:
206       - gerrit:
207           server-name: '{server-name}'
208           trigger-on:
209             - change-merged-event
210             - comment-added-contains-event:
211                 comment-contains-value: 'remerge'
212           projects:
213             - project-compare-type: 'ANT'
214               project-pattern: '{name}'
215               branches:
216                 - branch-compare-type: 'ANT'
217                   branch-pattern: '**/{branch}'
218           skip-vote:
219             successful: true
220             failed: true
221             unstable: true
222             notbuilt: true
223           # Force Jenkins always vote the values it should already have voted
224           # during the prior verify phase
225           override-votes: true
226           gerrit-build-started-verified-value: 1
227           gerrit-build-successful-verified-value: 1
228           gerrit-build-failed-verified-value: 1
229           gerrit-build-unstable-verified-value: 1
230           gerrit-build-notbuilt-verified-value: 1
231           gerrit-build-started-codereview-value: 0
232           gerrit-build-successful-codereview-value: 0
233           gerrit-build-failed-codereview-value: 0
234           gerrit-build-unstable-codereview-value: 0
235           gerrit-build-notbuilt-codereview-value: 0
236
237 # TODO: Unify argument names across gerrit-trigger-* macros.
238 - trigger:
239     name: gerrit-trigger-patch-sonar
240     triggers:
241       - gerrit:
242           server-name: '{server-name}'
243           trigger-on:
244             - comment-added-contains-event:
245                 comment-contains-value: 'run-sonar'
246           projects:
247             - project-compare-type: 'ANT'
248               project-pattern: '{name}'
249               branches:
250                 - branch-compare-type: 'ANT'
251                   branch-pattern: '**/master'
252           skip-vote:
253             successful: true
254             failed: true
255             unstable: true
256             notbuilt: true
257
258 - publisher:
259     name: archive-artifacts
260     publishers:
261       - archive:
262           artifacts: '{artifacts}'
263           allow-empty: true
264           fingerprint: true
265           latest-only: true
266
267 - publisher:
268     name: email-notification
269     publishers:
270       - email-ext:
271           recipients: '{email-recipients}'
272           reply-to: ''
273           content-type: default
274           subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
275           body: |
276               $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
277
278               Please refer to the logs server URL for console logs when possible
279               and use the Jenkins Build URL as a last resort.
280
281               Console Logs URL:
282               https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
283
284               Jenkins Build URL:
285               $BUILD_URL
286           unstable: true
287           fixed: true
288           send-to:
289             - recipients
290
291 - publisher:
292     name: jacoco-report
293     publishers:
294       - jacoco:
295           exec-pattern: "**/**.exec"
296           class-pattern: "**/classes"
297           source-pattern: "**/src/main/java"
298           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
299           status-update: true
300           targets:
301             - branch:
302                 healthy: 10
303                 unhealthy: 20
304             - method:
305                 healthy: 50
306                 unhealthy: 40
307
308 - builder:
309     # Deploys a maven site to Nexus using lftools nexus-zip command
310     name: opendaylight-infra-deploy-maven-site
311     builders:
312       - lf-provide-maven-settings:
313           global-settings-file: global-settings
314           settings-file: '{settings-file}'
315       - lf-infra-create-netrc:
316           server-id: opendaylight-site
317       - shell: !include-raw-escape:
318           - global-jjb/shell/lftools-install.sh
319           - opendaylight-infra-deploy-maven-site.sh
320       - lf-provide-maven-settings-cleanup
321
322 - builder:
323     name: opendaylight-infra-stack
324     # opendaylight-infra-stack.sh has a required variable {stack-template} that
325     # must be passed into this macro.
326     builders:
327       - shell: !include-raw: opendaylight-infra-stack.sh
328       - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh
329
330 - builder:
331     name: opendaylight-infra-push-gerrit-patch
332     # opendaylight-infra-push-gerrit-patch.sh allows a job to push a patch to
333     # Gerrit in an automated fashion. This is meant for tasks that creates
334     # the same patch regularly and needs the ability to detect if an unreviewed
335     # patch already exists. In which case it will update the existing patch.
336     #
337     # Note: This patch assumes the $WORKSPACE contains the project repo with
338     #       the files changed already "git add" and waiting for a "git commit" call.
339     #
340     # This script requires the following JJB variables to be passed in:
341     #     {project}       Gerrit project-name
342     #     {gerrit-topic}  Gerrit topic, please make a unique topic.
343     #     {gerrit-commit-message} Commit message to assign to commit.
344     # NOTE: Requires git review to be installed on node.
345     builders:
346       - shell: !include-raw: opendaylight-infra-push-gerrit-patch.sh
347
348 - builder:
349     name: distribute-build-url
350     # Place URL of the current run of a build job to a file at given path.
351     builders:
352       - shell: |
353           #!/bin/bash
354           set +e  # DO NOT fail script if command returns non-zero.
355
356           echo "$BUILD_URL" > {path}/build.url
357
358           # DO NOT fail the build if the echo failed.
359           exit 0
360
361 - builder:
362     name: wipe-org-opendaylight-repo
363     builders:
364       - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi'
365
366 - builder:
367     name: wipe-local-maven-repo
368     builders:
369       - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi'
370
371 - builder:
372     name: jacoco-nojava-workaround
373     builders:
374       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
375
376 - builder:
377     name: check-clm
378     builders:
379       - sonatype-clm:
380           application-name: '{application-name}'
381
382 - builder:
383     name: releng-check-unicode
384     builders:
385       - shell: |
386           $WORKSPACE/scripts/check-unicode.sh jjb/
387
388 - builder:
389     name: provide-maven-settings
390     builders:
391       - config-file-provider:
392           files:
393             - file-id: '{global-settings-file}'
394               variable: 'GLOBAL_SETTINGS_FILE'
395             - file-id: '{settings-file}'
396               variable: 'SETTINGS_FILE'
397
398 - builder:
399     name: releng-fetch-p2zip-if-necessary
400     builders:
401       - shell: |
402           # Cleanup any existing zips and metadata before we download the new update site
403           rm -f *.zip *.xml
404       - conditional-step:
405           condition-kind: strings-match
406           condition-string1: '$P2ZIP_URL'
407           condition-string2: ''
408           condition-basedir: workspace
409           steps:
410             # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT
411             - maven-target:
412                 maven-version: '{maven-version}'
413                 goals: >
414                     org.apache.maven.plugins:maven-dependency-plugin:get
415                     org.apache.maven.plugins:maven-dependency-plugin:copy
416                     -V -B
417                     -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip
418                     -DoutputDirectory=$WORKSPACE
419                 settings: '{settings}'
420                 global-settings: '{global-settings}'
421
422 - builder:
423     name: releng-generate-p2pom
424     builders:
425       - shell: !include-raw: include-raw-generate-p2pom.sh
426
427 - builder:
428     name: releng-update-p2composite-metadata
429     builders:
430       - shell: !include-raw: include-raw-update-p2composite-metadata.sh
431       - conditional-step:
432           condition-kind: file-exists
433           condition-filename: deploy-composite-repo.xml
434           condition-basedir: workspace
435           steps:
436             - maven-target:
437                 maven-version: '{maven-version}'
438                 pom: 'deploy-composite-repo.xml'
439                 goals: 'clean deploy -V -B -Dmaven.repo.local=/tmp/r'
440                 settings: '{settings}'
441                 global-settings: '{global-settings}'
442
443 - builder:
444     name: releng-stage-release
445     builders:
446       - shell: !include-raw: include-raw-stage-release.sh
447
448 - wrapper:
449     # This wrapper is required for all jobs as it configures the wrappers
450     # needed by OpenDaylight infra.
451     name: opendaylight-infra-wrappers
452     wrappers:
453       - mask-passwords
454       - config-file-provider:
455           files:
456             - file-id: npmrc
457               target: '$HOME/.npmrc'
458             - file-id: rackspace-heat
459               target: '$HOME/.config/openstack/clouds.yaml'
460       - timeout:
461           type: absolute
462           timeout: '{build-timeout}'
463           timeout-var: 'BUILD_TIMEOUT'
464           fail: true
465       - timestamps
466       - ssh-agent-credentials:
467           users:
468             - 'opendaylight-jenkins-ssh'
469       - openstack:
470           single-use: true
471
472 - builder:
473     name: packer-validate
474     builders:
475       - config-file-provider:
476           files:
477             - file-id: 'packer-cloud-env'
478               variable: 'CLOUDENV'
479       - shell: |
480           #!/bin/bash
481           cd packer
482           varfiles="../packer/vars/*"
483           templates="../packer/templates/*"
484           provision="../packer/provision/*.sh"
485           for v in $varfiles; do
486               [[ "${v##*/}" =~ ^(cloud-env.*)$ ]] && continue
487               for t in $templates; do
488                   export PACKER_LOG="yes" && \
489                   export PACKER_LOG_PATH="packer-validate-${v##*/}-${t##*/}.log" && \
490                               packer.io validate -var-file=$CLOUDENV \
491                               -var-file=$v $t
492                   if [ $? -ne 0 ]; then
493                       break
494                   fi
495               done
496           done
497
498 - builder:
499     name: packer-build
500     builders:
501       - config-file-provider:
502           files:
503             - file-id: 'packer-cloud-env'
504               variable: 'CLOUDENV'
505       - shell: |
506           #!/bin/bash
507           cd packer
508           export PACKER_LOG="yes" && \
509           export PACKER_LOG_PATH="packer-build.log" && \
510                       packer.io build -color=false -var-file=$CLOUDENV \
511                        -var-file=../packer/vars/{platform}.json \
512                        ../packer/templates/{template}.json
513           # Split public and private cloud logs
514           grep -e 'public_cloud' packer-build.log > packer-build_public_cloud.log  2>&1
515           grep -e 'private_cloud' packer-build.log > packer-build_private_cloud.log 2>&1
516
517 - builder:
518     # TODO: Verify signature after downloading users public key from a locally created
519     # repository instead of the public keymesh. This requires a process in place to get ODL
520     # developers public keys into a local repository without increasing the job thoughput.
521     name: verify-gpg-signature
522     builders:
523       - shell: !include-raw: include-raw-verify-gpg-signatures.sh
524
525 - builder:
526     name: opendaylight-infra-jjbini
527     builders:
528       - config-file-provider:
529           files:
530             - file-id: 'jjbini'
531               target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini'
532
533 - builder:
534     name: distribution-check-wipe
535     # Step zero: Wipe file repositories up front.
536     builders:
537       - shell: |
538           echo "wipe r: the local Maven repository"
539           rm -rfv /tmp/r
540           echo "wipe n: the fake remote (Nexus) repository"
541           rm -rfv /tmp/n
542           echo "wipe t: the transient repository used in some manipulations"
543           rm -rfv /tmp/t
544
545 - builder:
546     name: distribution-check-build-project
547     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
548     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
549     # Arguments:
550     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
551     builders:
552       - maven-target:
553           maven-version: 'mvn33'
554           pom: '{pom}'
555           goals: |
556               clean deploy dependency:tree
557               -DoutputFile=dependency_tree.txt
558               -Pq
559               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
560               {mvn-opts}
561           java-opts:
562             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
563           settings: 'integration-settings'
564           settings-type: cfp
565           global-settings: 'global-settings'
566           global-settings-type: cfp
567
568 - builder:
569     name: distribution-check-verify-groupid
570     # Step two: Verify all deployed artifacts belong to the project's groupId.
571     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
572     # The correct directory is derived from $GERRIT_PROJECT.
573     # TODO: Verify all deployed artifacts are snapshots.
574     # Arguments:
575     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
576     builders:
577       - shell: |
578           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
579           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
580           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
581           rm -rf /tmp/n
582           mv /tmp/t /tmp/n
583
584 - builder:
585     name: distribution-check-download-deps
586     # Step three: Online build of integration distribution.
587     # This step is mainly used for downloading other project artifacts.
588     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
589     # Arguments:
590     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
591     builders:
592       - maven-target:
593           maven-version: 'mvn33'
594           pom: '{dist-pom}'
595           goals: |
596               clean install dependency:tree
597               -DoutputFile=dependency_tree.txt
598               -Pq
599               {mvn-opts}
600           java-opts:
601             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
602           settings: 'integration-settings'
603           settings-type: cfp
604           global-settings: 'global-settings'
605           global-settings-type: cfp
606
607 - builder:
608     name: distribution-check-delete-snapshots
609     # Step four: Delete snapshot artifacts from the local repository.
610     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
611     # Also other files related to maven repository resolution are removed,
612     # and then empty directories are removed, in order to simplify debugging.
613     builders:
614       - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh
615
616 - builder:
617     name: distribution-check-configure-remotes
618     # Now the ugly part. It seems that the only way to tell Maven 2+
619     # which remote repositories to use is via settings.xml file.
620     # So we create such a file here, but it needs most of odlparent:settings.xml
621     builders:
622       - shell: |
623           echo '
624           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
625             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
626             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
627             https://maven.apache.org/xsd/settings-1.0.0.xsd">
628             <profiles>
629               <profile>
630                 <id>opendaylight-release</id>
631                 <repositories>
632                   <repository>
633                     <id>opendaylight-mirror</id>
634                     <name>opendaylight</name>
635                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
636                     <releases><updatePolicy>never</updatePolicy></releases>
637                     <snapshots><enabled>false</enabled></snapshots>
638                   </repository>
639                 </repositories>
640                 <pluginRepositories>
641                   <pluginRepository>
642                     <id>opendaylight-plugin-mirror</id>
643                     <name>opendaylight-plugin</name>
644                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
645                     <releases><updatePolicy>never</updatePolicy></releases>
646                     <snapshots><enabled>false</enabled></snapshots>
647                   </pluginRepository>
648                 </pluginRepositories>
649               </profile>
650               <profile>
651                 <id>file-snapshots</id>
652                 <repositories>
653                   <repository>
654                     <id>file-snapshots</id>
655                     <name>file</name>
656                     <url>file:///tmp/n/</url>
657                     <releases><enabled>false</enabled></releases>
658                   </repository>
659                 </repositories>
660                 <pluginRepositories>
661                   <pluginRepository>
662                     <id>file-plugin-snapshots</id>
663                     <name>file-plugin</name>
664                     <url>file:///tmp/n/</url>
665                     <releases><enabled>false</enabled></releases>
666                   </pluginRepository>
667                 </pluginRepositories>
668               </profile>
669             </profiles>
670             <activeProfiles>
671               <activeProfile>file-snapshots</activeProfile>
672               <activeProfile>opendaylight-release</activeProfile>
673             </activeProfiles>
674           </settings>
675           ' > fake_remotes.xml
676           # Notes: The settings are minimal in order to detect breakage scenarios,
677           # while allowing for the following quirks:
678           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
679           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
680           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
681
682 - builder:
683     name: distribution-check-repeat-build
684     # Step five: Repeat the distribution build but with the new settings.
685     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
686     # which faithfully reproduces conditions in later verify-like job runs.
687     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode"
688     # This is where SingleFeatureTest is not skipped.
689     # Arguments:
690     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
691     builders:
692       - maven-target:
693           maven-version: 'mvn33'
694           pom: '{dist-pom}'
695           goals: |
696               clean install dependency:tree
697               -DoutputFile=dependency_tree.txt -s fake_remotes.xml
698               -Pq
699               -DskipTests=false
700               {mvn-opts}
701           java-opts:
702             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'