Revert "Archive all *.txt and *.xml surefire"
[releng/builder.git] / jjb / releng-macros.yaml
1 # OLD Releng macros
2 ---
3 - property:
4     name: opendaylight-infra-properties
5     properties:
6       - build-discarder:
7           days-to-keep: '{build-days-to-keep}'
8           num-to-keep: 40
9           # Need to keep artifacts for at least 1 day as some projects need to
10           # be able to validate their artifacts and only allowing limited
11           # number of artifacts could make fast moving jobs lost their
12           # artifacts before the developers can test it.
13           artifact-days-to-keep: 1
14
15 - parameter:
16     name: opendaylight-infra-parameters
17     parameters:
18       - string:
19           name: PROJECT
20           default: '{project}'
21           description: 'Parameter to identify an ODL Gerrit project'
22       - string:
23           name: ARCHIVE_ARTIFACTS
24           default: >
25               {artifacts}
26               **/target/surefire-reports/*-output.txt
27               **/hs_err_*.log
28               **/target/feature/feature.xml
29               **/*.hprof
30           description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
31       - string:
32           name: GERRIT_PROJECT
33           default: '{project}'
34           description: "GERRIT_PROJECT parameter if not given by trigger"
35       - string:
36           name: GERRIT_BRANCH
37           default: '{branch}'
38           description: "JJB configured GERRIT_BRANCH parameter"
39       - string:
40           name: GERRIT_REFSPEC
41           default: '{refspec}'
42           description: "GERRIT_REFSPEC parameter if not given by trigger"
43       - string:
44           name: STACK_NAME
45           default: '$SILO-$JOB_NAME-$BUILD_NUMBER'
46           description: "Used by Heat to generate a unique stack & vm name"
47       - string:
48           name: OS_CLOUD
49           default: '{os-cloud}'
50           description: |
51               The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
52               variable name that is significant to openstack client as a
53               environment variable. Please refer to the documentation for
54               further details.
55               https://docs.openstack.org/developer/python-openstackclient/
56
57 - parameter:
58     name: build-tag
59     parameters:
60       - string:
61           name: BUILD_TAG
62           default: ''
63           description: 'Tag in Git to checkout'
64
65 - parameter:
66     name: controller-version-parameter
67     parameters:
68       - string:
69           name: ODL_VERSION
70           default: '{odl_version}'
71           description: 'Controller version (for use with openstacks networking_odl project)'
72
73 - parameter:
74     name: patches-to-build-parameter
75     parameters:
76       - string:
77           name: PATCHES_TO_BUILD
78           default: ''
79           description: |
80             Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47)
81
82 - parameter:
83     name: run-test-parameter
84     parameters:
85       - string:
86           name: RUNTEST
87           default: '{run-test}'
88           description: 'Set true to run test after build'
89
90 - parameter:
91     name: distribution-branch-to-build-parameter
92     parameters:
93       - string:
94           name: DISTRIBUTION_BRANCH_TO_BUILD
95           default: 'master'
96           description: 'distribution repo branch to build with'
97
98 - parameter:
99     name: p2zip-parameter
100     parameters:
101       - string:
102           name: P2ZIP_URL
103           default: ''
104           description: 'Nexus staging profile id'
105
106 - parameter:
107     name: stage-id-parameter
108     parameters:
109       - string:
110           name: STAGING_PROFILE_ID
111           default: '{stage-id}'
112           description: 'Nexus staging profile id'
113
114 - parameter:
115     name: maven-exec
116     parameters:
117       - string:
118           name: MVN
119           default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
120           description: 'Maven selector to be used by shell scripts'
121
122 - scm:
123     name: git-scm
124     scm:
125       - git:
126           credentials-id: 'opendaylight-jenkins-ssh'
127           url: '$GIT_BASE'
128           refspec: ''
129           branches:
130             - 'origin/{branch}'
131           skip-tag: true
132           wipe-workspace: true
133
134 - scm:
135     name: git-scm-with-submodules
136     scm:
137       - git:
138           credentials-id: 'opendaylight-jenkins-ssh'
139           url: '$GIT_BASE'
140           refspec: ''
141           branches:
142             - 'refs/heads/{branch}'
143           skip-tag: true
144           wipe-workspace: true
145           submodule:
146             recursive: true
147
148 - scm:
149     name: gerrit-trigger-scm
150     scm:
151       - git:
152           credentials-id: 'opendaylight-jenkins-ssh'
153           url: '$GIT_BASE'
154           refspec: '{refspec}'
155           branches:
156             - 'origin/$GERRIT_BRANCH'
157           skip-tag: true
158           choosing-strategy: '{choosing-strategy}'
159
160 - wrapper:
161     name: build-timeout
162     wrappers:
163       - timeout:
164           type: absolute
165           timeout: 360
166           fail: true
167
168 # This is a single macro to use for all jobs who vote on every (relevant) patch set.
169 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
170 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
171 # Arguments:
172 #     server: name of gerrit server to listen to
173 #     project: pattern to match triggering projects
174 #     branch: triggering branch name
175 #     files: pattern to match triggering filepaths
176 - trigger:
177     name: gerrit-trigger-patch-submitted
178     triggers:
179       - gerrit:
180           server-name: '{server}'
181           trigger-on:
182             - patchset-created-event:
183                 exclude-drafts: true
184                 exclude-trivial-rebase: false
185                 exclude-no-code-change: false
186             - draft-published-event
187             - comment-added-contains-event:
188                 comment-contains-value: recheck
189           projects:
190             - project-compare-type: ANT
191               project-pattern: '{project}'
192               branches:
193                 - branch-compare-type: ANT
194                   branch-pattern: '**/{branch}'
195               file-paths:
196                 - compare-type: ANT
197                   pattern: '{files}'
198
199 # TODO: Unify argument names across gerrit-trigger-* macros.
200 - trigger:
201     name: gerrit-trigger-patch-merged
202     triggers:
203       - gerrit:
204           server-name: '{server-name}'
205           trigger-on:
206             - change-merged-event
207             - comment-added-contains-event:
208                 comment-contains-value: 'remerge'
209           projects:
210             - project-compare-type: 'ANT'
211               project-pattern: '{name}'
212               branches:
213                 - branch-compare-type: 'ANT'
214                   branch-pattern: '**/{branch}'
215           skip-vote:
216             successful: true
217             failed: true
218             unstable: true
219             notbuilt: true
220           # Force Jenkins always vote the values it should already have voted
221           # during the prior verify phase
222           override-votes: true
223           gerrit-build-started-verified-value: 1
224           gerrit-build-successful-verified-value: 1
225           gerrit-build-failed-verified-value: 1
226           gerrit-build-unstable-verified-value: 1
227           gerrit-build-notbuilt-verified-value: 1
228           gerrit-build-started-codereview-value: 0
229           gerrit-build-successful-codereview-value: 0
230           gerrit-build-failed-codereview-value: 0
231           gerrit-build-unstable-codereview-value: 0
232           gerrit-build-notbuilt-codereview-value: 0
233
234 # TODO: Unify argument names across gerrit-trigger-* macros.
235 - trigger:
236     name: gerrit-trigger-patch-sonar
237     triggers:
238       - gerrit:
239           server-name: '{server-name}'
240           trigger-on:
241             - comment-added-contains-event:
242                 comment-contains-value: 'run-sonar'
243           projects:
244             - project-compare-type: 'ANT'
245               project-pattern: '{name}'
246               branches:
247                 - branch-compare-type: 'ANT'
248                   branch-pattern: '**/master'
249           skip-vote:
250             successful: true
251             failed: true
252             unstable: true
253             notbuilt: true
254
255 - publisher:
256     name: archive-artifacts
257     publishers:
258       - archive:
259           artifacts: '{artifacts}'
260           allow-empty: true
261           fingerprint: true
262           latest-only: true
263
264 - publisher:
265     name: email-notification
266     publishers:
267       - email-ext:
268           recipients: '{email-recipients}'
269           reply-to: ''
270           content-type: default
271           subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
272           body: |
273               $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
274
275               Please refer to the logs server URL for console logs when possible
276               and use the Jenkins Build URL as a last resort.
277
278               Console Logs URL:
279               https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
280
281               Jenkins Build URL:
282               $BUILD_URL
283           unstable: true
284           fixed: true
285           send-to:
286             - recipients
287
288 - publisher:
289     name: jacoco-report
290     publishers:
291       - jacoco:
292           exec-pattern: "**/**.exec"
293           class-pattern: "**/classes"
294           source-pattern: "**/src/main/java"
295           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
296           status-update: true
297           targets:
298             - branch:
299                 healthy: 10
300                 unhealthy: 20
301             - method:
302                 healthy: 50
303                 unhealthy: 40
304
305 - builder:
306     name: opendaylight-infra-stack
307     # opendaylight-infra-stack.sh has a required variable {stack-template} that
308     # must be passed into this macro.
309     builders:
310       - shell: !include-raw: opendaylight-infra-stack.sh
311       - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh
312
313 - builder:
314     name: opendaylight-infra-push-gerrit-patch
315     # opendaylight-infra-push-gerrit-patch.sh allows a job to push a patch to
316     # Gerrit in an automated fashion. This is meant for tasks that creates
317     # the same patch regularly and needs the ability to detect if an unreviewed
318     # patch already exists. In which case it will update the existing patch.
319     #
320     # Note: This patch assumes the $WORKSPACE contains the project repo with
321     #       the files changed already "git add" and waiting for a "git commit" call.
322     #
323     # This script requires the following JJB variables to be passed in:
324     #     {project}       Gerrit project-name
325     #     {gerrit-topic}  Gerrit topic, please make a unique topic.
326     #     {gerrit-commit-message} Commit message to assign to commit.
327     # NOTE: Requires git review to be installed on node.
328     builders:
329       - shell: !include-raw: opendaylight-infra-push-gerrit-patch.sh
330
331 - builder:
332     name: distribute-build-url
333     # Place URL of the current run of a build job to a file at given path.
334     builders:
335       - shell: |
336           #!/bin/bash
337           set +e  # DO NOT fail script if command returns non-zero.
338
339           echo "$BUILD_URL" > {path}/build.url
340
341           # DO NOT fail the build if the echo failed.
342           exit 0
343
344 - builder:
345     name: wipe-org-opendaylight-repo
346     builders:
347       - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi'
348
349 - builder:
350     name: wipe-local-maven-repo
351     builders:
352       - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi'
353
354 - builder:
355     name: jacoco-nojava-workaround
356     builders:
357       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
358
359 - builder:
360     name: check-clm
361     builders:
362       - sonatype-clm:
363           application-name: '{application-name}'
364
365 - builder:
366     name: releng-check-unicode
367     builders:
368       - shell: |
369           $WORKSPACE/scripts/check-unicode.sh jjb/
370
371 - builder:
372     name: provide-maven-settings
373     builders:
374       - config-file-provider:
375           files:
376             - file-id: '{global-settings-file}'
377               variable: 'GLOBAL_SETTINGS_FILE'
378             - file-id: '{settings-file}'
379               variable: 'SETTINGS_FILE'
380
381 - builder:
382     name: releng-fetch-p2zip-if-necessary
383     builders:
384       - shell: |
385           # Cleanup any existing zips and metadata before we download the new update site
386           rm -f *.zip *.xml
387       - conditional-step:
388           condition-kind: strings-match
389           condition-string1: '$P2ZIP_URL'
390           condition-string2: ''
391           condition-basedir: workspace
392           steps:
393             # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT
394             - maven-target:
395                 maven-version: '{maven-version}'
396                 goals: >
397                     org.apache.maven.plugins:maven-dependency-plugin:get
398                     org.apache.maven.plugins:maven-dependency-plugin:copy
399                     -V -B
400                     -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip
401                     -DoutputDirectory=$WORKSPACE
402                 settings: '{settings}'
403                 global-settings: '{global-settings}'
404
405 - builder:
406     name: releng-generate-p2pom
407     builders:
408       - shell: !include-raw: include-raw-generate-p2pom.sh
409
410 - builder:
411     name: releng-update-p2composite-metadata
412     builders:
413       - shell: !include-raw: include-raw-update-p2composite-metadata.sh
414       - conditional-step:
415           condition-kind: file-exists
416           condition-filename: deploy-composite-repo.xml
417           condition-basedir: workspace
418           steps:
419             - maven-target:
420                 maven-version: '{maven-version}'
421                 pom: 'deploy-composite-repo.xml'
422                 goals: 'clean deploy -V -B -Dmaven.repo.local=/tmp/r'
423                 settings: '{settings}'
424                 global-settings: '{global-settings}'
425
426 - builder:
427     name: releng-stage-release
428     builders:
429       - shell: !include-raw: include-raw-stage-release.sh
430
431 - wrapper:
432     # This wrapper is required for all jobs as it configures the wrappers
433     # needed by OpenDaylight infra.
434     name: opendaylight-infra-wrappers
435     wrappers:
436       - mask-passwords
437       - config-file-provider:
438           files:
439             - file-id: npmrc
440               target: '$HOME/.npmrc'
441             - file-id: rackspace-heat
442               target: '$HOME/.config/openstack/clouds.yaml'
443       - timeout:
444           type: absolute
445           timeout: '{build-timeout}'
446           timeout-var: 'BUILD_TIMEOUT'
447           fail: true
448       - timestamps
449       - ssh-agent-credentials:
450           users:
451             - 'opendaylight-jenkins-ssh'
452       - openstack:
453           single-use: true
454
455 - builder:
456     name: packer-validate
457     builders:
458       - config-file-provider:
459           files:
460             - file-id: 'packer-cloud-env'
461               variable: 'CLOUDENV'
462       - shell: |
463           #!/bin/bash
464           cd packer
465           varfiles="../packer/vars/*"
466           templates="../packer/templates/*"
467           provision="../packer/provision/*.sh"
468           for v in $varfiles; do
469               [[ "${v##*/}" =~ ^(cloud-env.*)$ ]] && continue
470               for t in $templates; do
471                   export PACKER_LOG="yes" && \
472                   export PACKER_LOG_PATH="packer-validate-${v##*/}-${t##*/}.log" && \
473                               packer.io validate -var-file=$CLOUDENV \
474                               -var-file=$v $t
475                   if [ $? -ne 0 ]; then
476                       break
477                   fi
478               done
479           done
480
481 - builder:
482     name: packer-build
483     builders:
484       - config-file-provider:
485           files:
486             - file-id: 'packer-cloud-env'
487               variable: 'CLOUDENV'
488       - shell: |
489           #!/bin/bash
490           cd packer
491           export PACKER_LOG="yes" && \
492           export PACKER_LOG_PATH="packer-build.log" && \
493                       packer.io build -color=false -var-file=$CLOUDENV \
494                        -var-file=../packer/vars/{platform}.json \
495                        ../packer/templates/{template}.json
496           # Split public and private cloud logs
497           grep -e 'public_cloud' packer-build.log > packer-build_public_cloud.log  2>&1
498           grep -e 'private_cloud' packer-build.log > packer-build_private_cloud.log 2>&1
499
500 - builder:
501     # TODO: Verify signature after downloading users public key from a locally created
502     # repository instead of the public keymesh. This requires a process in place to get ODL
503     # developers public keys into a local repository without increasing the job thoughput.
504     name: verify-gpg-signature
505     builders:
506       - shell: !include-raw: include-raw-verify-gpg-signatures.sh
507
508 - builder:
509     name: opendaylight-infra-jjbini
510     builders:
511       - config-file-provider:
512           files:
513             - file-id: 'jjbini'
514               target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini'
515
516 - builder:
517     name: distribution-check-wipe
518     # Step zero: Wipe file repositories up front.
519     builders:
520       - shell: |
521           echo "wipe r: the local Maven repository"
522           rm -rfv /tmp/r
523           echo "wipe n: the fake remote (Nexus) repository"
524           rm -rfv /tmp/n
525           echo "wipe t: the transient repository used in some manipulations"
526           rm -rfv /tmp/t
527
528 - builder:
529     name: distribution-check-build-project
530     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
531     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
532     # Arguments:
533     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
534     builders:
535       - maven-target:
536           maven-version: 'mvn33'
537           pom: '{pom}'
538           goals: |
539               clean deploy dependency:tree
540               -DoutputFile=dependency_tree.txt
541               -Pq
542               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
543               {mvn-opts}
544           java-opts:
545             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
546           settings: 'integration-settings'
547           settings-type: cfp
548           global-settings: 'global-settings'
549           global-settings-type: cfp
550
551 - builder:
552     name: distribution-check-verify-groupid
553     # Step two: Verify all deployed artifacts belong to the project's groupId.
554     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
555     # The correct directory is derived from $GERRIT_PROJECT.
556     # TODO: Verify all deployed artifacts are snapshots.
557     # Arguments:
558     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
559     builders:
560       - shell: |
561           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
562           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
563           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
564           rm -rf /tmp/n
565           mv /tmp/t /tmp/n
566
567 - builder:
568     name: distribution-check-download-deps
569     # Step three: Online build of integration distribution.
570     # This step is mainly used for downloading other project artifacts.
571     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
572     # Arguments:
573     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
574     builders:
575       - maven-target:
576           maven-version: 'mvn33'
577           pom: '{dist-pom}'
578           goals: |
579               clean install dependency:tree
580               -DoutputFile=dependency_tree.txt
581               -Pq
582               {mvn-opts}
583           java-opts:
584             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
585           settings: 'integration-settings'
586           settings-type: cfp
587           global-settings: 'global-settings'
588           global-settings-type: cfp
589
590 - builder:
591     name: distribution-check-delete-snapshots
592     # Step four: Delete snapshot artifacts from the local repository.
593     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
594     # Also other files related to maven repository resolution are removed,
595     # and then empty directories are removed, in order to simplify debugging.
596     builders:
597       - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh
598
599 - builder:
600     name: distribution-check-configure-remotes
601     # Now the ugly part. It seems that the only way to tell Maven 2+
602     # which remote repositories to use is via settings.xml file.
603     # So we create such a file here, but it needs most of odlparent:settings.xml
604     builders:
605       - shell: |
606           echo '
607           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
608             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
609             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
610             https://maven.apache.org/xsd/settings-1.0.0.xsd">
611             <profiles>
612               <profile>
613                 <id>opendaylight-release</id>
614                 <repositories>
615                   <repository>
616                     <id>opendaylight-mirror</id>
617                     <name>opendaylight</name>
618                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
619                     <releases><updatePolicy>never</updatePolicy></releases>
620                     <snapshots><enabled>false</enabled></snapshots>
621                   </repository>
622                 </repositories>
623                 <pluginRepositories>
624                   <pluginRepository>
625                     <id>opendaylight-plugin-mirror</id>
626                     <name>opendaylight-plugin</name>
627                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
628                     <releases><updatePolicy>never</updatePolicy></releases>
629                     <snapshots><enabled>false</enabled></snapshots>
630                   </pluginRepository>
631                 </pluginRepositories>
632               </profile>
633               <profile>
634                 <id>file-snapshots</id>
635                 <repositories>
636                   <repository>
637                     <id>file-snapshots</id>
638                     <name>file</name>
639                     <url>file:///tmp/n/</url>
640                     <releases><enabled>false</enabled></releases>
641                   </repository>
642                 </repositories>
643                 <pluginRepositories>
644                   <pluginRepository>
645                     <id>file-plugin-snapshots</id>
646                     <name>file-plugin</name>
647                     <url>file:///tmp/n/</url>
648                     <releases><enabled>false</enabled></releases>
649                   </pluginRepository>
650                 </pluginRepositories>
651               </profile>
652             </profiles>
653             <activeProfiles>
654               <activeProfile>file-snapshots</activeProfile>
655               <activeProfile>opendaylight-release</activeProfile>
656             </activeProfiles>
657           </settings>
658           ' > fake_remotes.xml
659           # Notes: The settings are minimal in order to detect breakage scenarios,
660           # while allowing for the following quirks:
661           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
662           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
663           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
664
665 - builder:
666     name: distribution-check-repeat-build
667     # Step five: Repeat the distribution build but with the new settings.
668     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
669     # which faithfully reproduces conditions in later verify-like job runs.
670     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode"
671     # This is where SingleFeatureTest is not skipped.
672     # Arguments:
673     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
674     builders:
675       - maven-target:
676           maven-version: 'mvn33'
677           pom: '{dist-pom}'
678           goals: |
679               clean install dependency:tree
680               -DoutputFile=dependency_tree.txt -s fake_remotes.xml
681               -Pq
682               -DskipTests=false
683               {mvn-opts}
684           java-opts:
685             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'