Add step to grab csit test status and archive it
[releng/builder.git] / jjb / releng-macros.yaml
1 # OLD Releng macros
2 ---
3 - property:
4     name: opendaylight-infra-properties
5     properties:
6       - build-discarder:
7           days-to-keep: '{build-days-to-keep}'
8           num-to-keep: 40
9           # Need to keep artifacts for at least 1 day as some projects need to
10           # be able to validate their artifacts and only allowing limited
11           # number of artifacts could make fast moving jobs lost their
12           # artifacts before the developers can test it.
13           artifact-days-to-keep: 1
14
15 - parameter:
16     name: opendaylight-infra-parameters
17     parameters:
18       - string:
19           name: PROJECT
20           default: '{project}'
21           description: 'Parameter to identify an ODL Gerrit project'
22       - string:
23           name: ARCHIVE_ARTIFACTS
24           # Before adding more here, beware of https://jira.linuxfoundation.org/browse/RELENG-280
25           default: >
26               {artifacts}
27               **/target/surefire-reports/*-output.txt
28               **/target/failsafe-reports/failsafe-summary.xml
29               **/hs_err_*.log
30               **/target/feature/feature.xml
31               **/*.hprof
32           description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
33       - string:
34           name: GERRIT_PROJECT
35           default: '{project}'
36           description: "GERRIT_PROJECT parameter if not given by trigger"
37       - string:
38           name: GERRIT_BRANCH
39           default: '{branch}'
40           description: "JJB configured GERRIT_BRANCH parameter"
41       - string:
42           name: GERRIT_REFSPEC
43           default: '{refspec}'
44           description: |
45               Gerrit reference to checkout (ex: refs/heads/master,
46               refs/changes/48/61548/1 where 61548 is Gerrit change ID, 1 is
47               patch number and 48 is last two digits of change ID)
48       - string:
49           name: STACK_NAME
50           default: '$SILO-$JOB_NAME-$BUILD_NUMBER'
51           description: "Used by Heat to generate a unique stack & vm name"
52       - string:
53           name: OS_CLOUD
54           default: '{os-cloud}'
55           description: |
56               The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
57               variable name that is significant to openstack client as a
58               environment variable. Please refer to the documentation for
59               further details.
60               https://docs.openstack.org/developer/python-openstackclient/
61
62 - parameter:
63     name: build-tag
64     parameters:
65       - string:
66           name: BUILD_TAG
67           default: ''
68           description: 'Tag in Git to checkout'
69
70 - parameter:
71     name: controller-version-parameter
72     parameters:
73       - string:
74           name: ODL_VERSION
75           default: '{odl_version}'
76           description: 'Controller version (for use with openstacks networking_odl project)'
77
78 - parameter:
79     name: patches-to-build-parameter
80     parameters:
81       - string:
82           name: PATCHES_TO_BUILD
83           default: ''
84           description: |
85             Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47)
86
87 - parameter:
88     name: run-test-parameter
89     parameters:
90       - string:
91           name: RUNTEST
92           default: '{run-test}'
93           description: 'Set true to run test after build'
94
95 - parameter:
96     name: distribution-branch-to-build-parameter
97     parameters:
98       - string:
99           name: DISTRIBUTION_BRANCH_TO_BUILD
100           default: 'master'
101           description: 'distribution repo branch to build with'
102
103 - parameter:
104     name: p2zip-parameter
105     parameters:
106       - string:
107           name: P2ZIP_URL
108           default: ''
109           description: 'Nexus staging profile id'
110
111 - parameter:
112     name: stage-id-parameter
113     parameters:
114       - string:
115           name: STAGING_PROFILE_ID
116           default: '{stage-id}'
117           description: 'Nexus staging profile id'
118
119 - parameter:
120     name: maven-exec
121     parameters:
122       - string:
123           name: MVN
124           default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
125           description: 'Maven selector to be used by shell scripts'
126
127 - scm:
128     name: git-scm
129     scm:
130       - git:
131           credentials-id: 'opendaylight-jenkins-ssh'
132           url: '$GIT_BASE'
133           refspec: ''
134           branches:
135             - 'origin/{branch}'
136           skip-tag: true
137           wipe-workspace: true
138
139 - scm:
140     name: git-scm-with-submodules
141     scm:
142       - git:
143           credentials-id: 'opendaylight-jenkins-ssh'
144           url: '$GIT_BASE'
145           refspec: ''
146           branches:
147             - 'refs/heads/{branch}'
148           skip-tag: true
149           wipe-workspace: true
150           submodule:
151             recursive: true
152
153 - scm:
154     name: gerrit-trigger-scm
155     scm:
156       - git:
157           credentials-id: 'opendaylight-jenkins-ssh'
158           url: '$GIT_BASE'
159           refspec: '{refspec}'
160           branches:
161             - 'origin/$GERRIT_BRANCH'
162           skip-tag: true
163           choosing-strategy: '{choosing-strategy}'
164
165 - wrapper:
166     name: build-timeout
167     wrappers:
168       - timeout:
169           type: absolute
170           timeout: 360
171           fail: true
172
173 # This is a single macro to use for all jobs who vote on every (relevant) patch set.
174 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
175 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
176 # Arguments:
177 #     server: name of gerrit server to listen to
178 #     project: pattern to match triggering projects
179 #     branch: triggering branch name
180 #     files: pattern to match triggering filepaths
181 - trigger:
182     name: gerrit-trigger-patch-submitted
183     triggers:
184       - gerrit:
185           server-name: '{server}'
186           trigger-on:
187             - patchset-created-event:
188                 exclude-drafts: true
189                 exclude-trivial-rebase: false
190                 exclude-no-code-change: false
191             - draft-published-event
192             - comment-added-contains-event:
193                 comment-contains-value: recheck
194           projects:
195             - project-compare-type: ANT
196               project-pattern: '{project}'
197               branches:
198                 - branch-compare-type: ANT
199                   branch-pattern: '**/{branch}'
200               file-paths:
201                 - compare-type: ANT
202                   pattern: '{files}'
203
204 # TODO: Unify argument names across gerrit-trigger-* macros.
205 - trigger:
206     name: gerrit-trigger-patch-merged
207     triggers:
208       - gerrit:
209           server-name: '{server-name}'
210           trigger-on:
211             - change-merged-event
212             - comment-added-contains-event:
213                 comment-contains-value: 'remerge'
214           projects:
215             - project-compare-type: 'ANT'
216               project-pattern: '{name}'
217               branches:
218                 - branch-compare-type: 'ANT'
219                   branch-pattern: '**/{branch}'
220           skip-vote:
221             successful: true
222             failed: true
223             unstable: true
224             notbuilt: true
225           # Force Jenkins always vote the values it should already have voted
226           # during the prior verify phase
227           override-votes: true
228           gerrit-build-started-verified-value: 1
229           gerrit-build-successful-verified-value: 1
230           gerrit-build-failed-verified-value: 1
231           gerrit-build-unstable-verified-value: 1
232           gerrit-build-notbuilt-verified-value: 1
233           gerrit-build-started-codereview-value: 0
234           gerrit-build-successful-codereview-value: 0
235           gerrit-build-failed-codereview-value: 0
236           gerrit-build-unstable-codereview-value: 0
237           gerrit-build-notbuilt-codereview-value: 0
238
239 # TODO: Unify argument names across gerrit-trigger-* macros.
240 - trigger:
241     name: gerrit-trigger-patch-sonar
242     triggers:
243       - gerrit:
244           server-name: '{server-name}'
245           trigger-on:
246             - comment-added-contains-event:
247                 comment-contains-value: 'run-sonar'
248           projects:
249             - project-compare-type: 'ANT'
250               project-pattern: '{name}'
251               branches:
252                 - branch-compare-type: 'ANT'
253                   branch-pattern: '**/master'
254           skip-vote:
255             successful: true
256             failed: true
257             unstable: true
258             notbuilt: true
259
260 - publisher:
261     name: email-notification
262     publishers:
263       - email-ext:
264           recipients: '{email-recipients}'
265           reply-to: ''
266           content-type: default
267           subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
268           body: |
269               $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
270
271               Please refer to the logs server URL for console logs when possible
272               and use the Jenkins Build URL as a last resort.
273
274               Console Logs URL:
275               https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
276
277               Jenkins Build URL:
278               $BUILD_URL
279           unstable: true
280           fixed: true
281           send-to:
282             - recipients
283
284 - publisher:
285     name: jacoco-report
286     publishers:
287       - jacoco:
288           exec-pattern: "**/**.exec"
289           class-pattern: "**/classes"
290           source-pattern: "**/src/main/java"
291           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
292           status-update: true
293           targets:
294             - branch:
295                 healthy: 10
296                 unhealthy: 20
297             - method:
298                 healthy: 50
299                 unhealthy: 40
300
301 - builder:
302     # Deploys a maven site to Nexus using lftools nexus-zip command
303     name: opendaylight-infra-deploy-maven-site
304     builders:
305       - lf-provide-maven-settings:
306           global-settings-file: global-settings
307           settings-file: '{settings-file}'
308       - lf-infra-create-netrc:
309           server-id: opendaylight-site
310       - shell: !include-raw-escape:
311           - global-jjb/shell/lftools-install.sh
312           - opendaylight-infra-deploy-maven-site.sh
313       - lf-provide-maven-settings-cleanup
314
315 - builder:
316     name: opendaylight-infra-stack
317     # opendaylight-infra-stack.sh has a required variable {stack-template} that
318     # must be passed into this macro.
319     builders:
320       - shell: !include-raw: opendaylight-infra-stack.sh
321       - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh
322
323 - builder:
324     name: opendaylight-infra-push-gerrit-patch
325     # opendaylight-infra-push-gerrit-patch.sh allows a job to push a patch to
326     # Gerrit in an automated fashion. This is meant for tasks that creates
327     # the same patch regularly and needs the ability to detect if an unreviewed
328     # patch already exists. In which case it will update the existing patch.
329     #
330     # Note: This patch assumes the $WORKSPACE contains the project repo with
331     #       the files changed already "git add" and waiting for a "git commit" call.
332     #
333     # This script requires the following JJB variables to be passed in:
334     #     {project}       Gerrit project-name
335     #     {gerrit-topic}  Gerrit topic, please make a unique topic.
336     #     {gerrit-commit-message} Commit message to assign to commit.
337     # NOTE: Requires git review to be installed on node.
338     builders:
339       - shell: !include-raw: opendaylight-infra-push-gerrit-patch.sh
340
341 - builder:
342     name: distribute-build-url
343     # Place URL of the current run of a build job to a file at given path.
344     builders:
345       - shell: |
346           #!/bin/bash
347           set +e  # DO NOT fail script if command returns non-zero.
348
349           echo "$BUILD_URL" > {path}/build.url
350
351           # DO NOT fail the build if the echo failed.
352           exit 0
353
354 - builder:
355     name: wipe-org-opendaylight-repo
356     builders:
357       - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi'
358
359 - builder:
360     name: wipe-local-maven-repo
361     builders:
362       - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi'
363
364 - builder:
365     name: jacoco-nojava-workaround
366     builders:
367       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
368
369 - builder:
370     name: check-clm
371     builders:
372       - sonatype-clm:
373           application-name: '{application-name}'
374
375 - builder:
376     name: releng-check-unicode
377     builders:
378       - shell: |
379           $WORKSPACE/scripts/check-unicode.sh jjb/
380
381 - builder:
382     name: provide-maven-settings
383     builders:
384       - config-file-provider:
385           files:
386             - file-id: '{global-settings-file}'
387               variable: 'GLOBAL_SETTINGS_FILE'
388             - file-id: '{settings-file}'
389               variable: 'SETTINGS_FILE'
390
391 - builder:
392     name: releng-fetch-p2zip-if-necessary
393     builders:
394       - shell: |
395           # Cleanup any existing zips and metadata before we download the new update site
396           rm -f *.zip *.xml
397       - conditional-step:
398           condition-kind: strings-match
399           condition-string1: '$P2ZIP_URL'
400           condition-string2: ''
401           condition-basedir: workspace
402           steps:
403             # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT
404             - maven-target:
405                 maven-version: '{maven-version}'
406                 goals: >
407                     org.apache.maven.plugins:maven-dependency-plugin:get
408                     org.apache.maven.plugins:maven-dependency-plugin:copy
409                     -V -B
410                     -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip
411                     -DoutputDirectory=$WORKSPACE
412                 settings: '{settings}'
413                 global-settings: '{global-settings}'
414
415 - builder:
416     name: releng-generate-p2pom
417     builders:
418       - shell: !include-raw: generate-p2pom.sh
419
420 - builder:
421     name: releng-update-p2composite-metadata
422     builders:
423       - shell: !include-raw: update-p2composite-metadata.sh
424       - conditional-step:
425           condition-kind: file-exists
426           condition-filename: deploy-composite-repo.xml
427           condition-basedir: workspace
428           steps:
429             - maven-target:
430                 maven-version: '{maven-version}'
431                 pom: 'deploy-composite-repo.xml'
432                 goals: 'clean deploy -V -B -Dmaven.repo.local=/tmp/r'
433                 settings: '{settings}'
434                 global-settings: '{global-settings}'
435
436 - builder:
437     name: releng-stage-release
438     builders:
439       - shell: !include-raw: stage-release.sh
440
441 - wrapper:
442     # This wrapper is required for all jobs as it configures the wrappers
443     # needed by OpenDaylight infra.
444     name: opendaylight-infra-wrappers
445     wrappers:
446       - mask-passwords
447       - config-file-provider:
448           files:
449             - file-id: npmrc
450               target: '$HOME/.npmrc'
451             - file-id: rackspace-heat
452               target: '$HOME/.config/openstack/clouds.yaml'
453       - timeout:
454           type: absolute
455           timeout: '{build-timeout}'
456           timeout-var: 'BUILD_TIMEOUT'
457           fail: true
458       - timestamps
459       - ssh-agent-credentials:
460           users:
461             - 'opendaylight-jenkins-ssh'
462       - openstack:
463           single-use: true
464
465 - builder:
466     # TODO: Verify signature after downloading users public key from a locally created
467     # repository instead of the public keymesh. This requires a process in place to get ODL
468     # developers public keys into a local repository without increasing the job thoughput.
469     name: verify-gpg-signature
470     builders:
471       - shell: !include-raw: verify-gpg-signatures.sh
472
473 - builder:
474     name: opendaylight-infra-jjbini
475     builders:
476       - config-file-provider:
477           files:
478             - file-id: 'jjbini'
479               target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini'
480
481 - builder:
482     name: distribution-check-wipe
483     # Step zero: Wipe file repositories up front.
484     builders:
485       - shell: |
486           echo "wipe r: the local Maven repository"
487           rm -rfv /tmp/r
488           echo "wipe n: the fake remote (Nexus) repository"
489           rm -rfv /tmp/n
490           echo "wipe t: the transient repository used in some manipulations"
491           rm -rfv /tmp/t
492
493 - builder:
494     name: distribution-check-build-project
495     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
496     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
497     # Arguments:
498     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
499     builders:
500       - maven-target:
501           maven-version: 'mvn33'
502           pom: '{pom}'
503           goals: |
504               clean deploy dependency:tree
505               -DoutputFile=dependency_tree.txt
506               -Pq
507               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
508               {mvn-opts}
509           java-opts:
510             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
511           settings: 'integration-settings'
512           settings-type: cfp
513           global-settings: 'global-settings'
514           global-settings-type: cfp
515
516 - builder:
517     name: distribution-check-verify-groupid
518     # Step two: Verify all deployed artifacts belong to the project's groupId.
519     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
520     # The correct directory is derived from $GERRIT_PROJECT.
521     # TODO: Verify all deployed artifacts are snapshots.
522     # Arguments:
523     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
524     builders:
525       - shell: |
526           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
527           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
528           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
529           rm -rf /tmp/n
530           mv /tmp/t /tmp/n
531
532 - builder:
533     name: distribution-check-download-deps
534     # Step three: Online build of integration distribution.
535     # This step is mainly used for downloading other project artifacts.
536     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
537     # Arguments:
538     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
539     builders:
540       - maven-target:
541           maven-version: 'mvn33'
542           pom: '{dist-pom}'
543           goals: |
544               clean install dependency:tree
545               -DoutputFile=dependency_tree.txt
546               -Pq
547               {mvn-opts}
548           java-opts:
549             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
550           settings: 'integration-settings'
551           settings-type: cfp
552           global-settings: 'global-settings'
553           global-settings-type: cfp
554
555 - builder:
556     name: distribution-check-delete-snapshots
557     # Step four: Delete snapshot artifacts from the local repository.
558     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
559     # Also other files related to maven repository resolution are removed,
560     # and then empty directories are removed, in order to simplify debugging.
561     builders:
562       - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh
563
564 - builder:
565     name: distribution-check-configure-remotes
566     # Now the ugly part. It seems that the only way to tell Maven 2+
567     # which remote repositories to use is via settings.xml file.
568     # So we create such a file here, but it needs most of odlparent:settings.xml
569     builders:
570       - shell: |
571           echo '
572           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
573             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
574             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
575             https://maven.apache.org/xsd/settings-1.0.0.xsd">
576             <profiles>
577               <profile>
578                 <id>opendaylight-release</id>
579                 <repositories>
580                   <repository>
581                     <id>opendaylight-mirror</id>
582                     <name>opendaylight</name>
583                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
584                     <releases><updatePolicy>never</updatePolicy></releases>
585                     <snapshots><enabled>false</enabled></snapshots>
586                   </repository>
587                 </repositories>
588                 <pluginRepositories>
589                   <pluginRepository>
590                     <id>opendaylight-plugin-mirror</id>
591                     <name>opendaylight-plugin</name>
592                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
593                     <releases><updatePolicy>never</updatePolicy></releases>
594                     <snapshots><enabled>false</enabled></snapshots>
595                   </pluginRepository>
596                 </pluginRepositories>
597               </profile>
598               <profile>
599                 <id>file-snapshots</id>
600                 <repositories>
601                   <repository>
602                     <id>file-snapshots</id>
603                     <name>file</name>
604                     <url>file:///tmp/n/</url>
605                     <releases><enabled>false</enabled></releases>
606                   </repository>
607                 </repositories>
608                 <pluginRepositories>
609                   <pluginRepository>
610                     <id>file-plugin-snapshots</id>
611                     <name>file-plugin</name>
612                     <url>file:///tmp/n/</url>
613                     <releases><enabled>false</enabled></releases>
614                   </pluginRepository>
615                 </pluginRepositories>
616               </profile>
617             </profiles>
618             <activeProfiles>
619               <activeProfile>file-snapshots</activeProfile>
620               <activeProfile>opendaylight-release</activeProfile>
621             </activeProfiles>
622           </settings>
623           ' > fake_remotes.xml
624           # Notes: The settings are minimal in order to detect breakage scenarios,
625           # while allowing for the following quirks:
626           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
627           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
628           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
629
630 - builder:
631     name: distribution-check-repeat-build
632     # Step five: Repeat the distribution build but with the new settings.
633     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
634     # which faithfully reproduces conditions in later verify-like job runs.
635     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode"
636     # This is where SingleFeatureTest is not skipped.
637     # Arguments:
638     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
639     builders:
640       - maven-target:
641           maven-version: 'mvn33'
642           pom: '{dist-pom}'
643           goals: |
644               clean install dependency:tree
645               -DoutputFile=dependency_tree.txt -s fake_remotes.xml
646               -Pq
647               -Dsft.heap.max=4g
648               -DskipTests=false
649               {mvn-opts}
650           java-opts:
651             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
652
653 - publisher:
654     name: opendaylight-infra-generate-csit-status-report
655     publishers:
656       - postbuildscript:
657           builders:
658             - shell: !include-raw-escape:
659                 - generate-csit-status-report.sh
660           script-only-if-succeeded: false
661           script-only-if-failed: false
662           mark-unstable-if-failed: false