Merge "Update default RPM snap build version to Nitrogen"
[releng/builder.git] / jjb / releng-macros.yaml
1 # OLD Releng macros
2 ---
3 - property:
4     name: opendaylight-infra-properties
5     properties:
6       - build-discarder:
7           days-to-keep: '{build-days-to-keep}'
8           num-to-keep: 40
9           # Need to keep artifacts for at least 1 day as some projects need to
10           # be able to validate their artifacts and only allowing limited
11           # number of artifacts could make fast moving jobs lost their
12           # artifacts before the developers can test it.
13           artifact-days-to-keep: 1
14
15 - parameter:
16     name: opendaylight-infra-parameters
17     parameters:
18       - string:
19           name: PROJECT
20           default: '{project}'
21           description: 'Parameter to identify an ODL Gerrit project'
22       - string:
23           name: ARCHIVE_ARTIFACTS
24           default: >
25               {artifacts}
26               **/target/surefire-reports/*-output.txt
27               **/hs_err_*.log
28               **/target/feature/feature.xml
29               **/*.hprof
30           description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
31       - string:
32           name: GERRIT_PROJECT
33           default: '{project}'
34           description: "GERRIT_PROJECT parameter if not given by trigger"
35       - string:
36           name: GERRIT_BRANCH
37           default: '{branch}'
38           description: "JJB configured GERRIT_BRANCH parameter"
39       - string:
40           name: GERRIT_REFSPEC
41           default: '{refspec}'
42           description: "GERRIT_REFSPEC parameter if not given by trigger"
43       - string:
44           name: STACK_NAME
45           default: '$SILO-$JOB_NAME-$BUILD_NUMBER'
46           description: "Used by Heat to generate a unique stack & vm name"
47       - string:
48           name: OS_CLOUD
49           default: '{os-cloud}'
50           description: |
51               The name of a cloud configuration in clouds.yaml. OS_CLOUD is a
52               variable name that is significant to openstack client as a
53               environment variable. Please refer to the documentation for
54               further details.
55               https://docs.openstack.org/developer/python-openstackclient/
56
57 - parameter:
58     name: build-tag
59     parameters:
60       - string:
61           name: BUILD_TAG
62           default: ''
63           description: 'Tag in Git to checkout'
64
65 - parameter:
66     name: controller-version-parameter
67     parameters:
68       - string:
69           name: ODL_VERSION
70           default: '{odl_version}'
71           description: 'Controller version (for use with openstacks networking_odl project)'
72
73 - parameter:
74     name: patches-to-build-parameter
75     parameters:
76       - string:
77           name: PATCHES_TO_BUILD
78           default: ''
79           description: |
80             Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47)
81
82 - parameter:
83     name: run-test-parameter
84     parameters:
85       - string:
86           name: RUNTEST
87           default: '{run-test}'
88           description: 'Set true to run test after build'
89
90 - parameter:
91     name: distribution-branch-to-build-parameter
92     parameters:
93       - string:
94           name: DISTRIBUTION_BRANCH_TO_BUILD
95           default: 'master'
96           description: 'distribution repo branch to build with'
97
98 - parameter:
99     name: p2zip-parameter
100     parameters:
101       - string:
102           name: P2ZIP_URL
103           default: ''
104           description: 'Nexus staging profile id'
105
106 - parameter:
107     name: stage-id-parameter
108     parameters:
109       - string:
110           name: STAGING_PROFILE_ID
111           default: '{stage-id}'
112           description: 'Nexus staging profile id'
113
114 - parameter:
115     name: maven-exec
116     parameters:
117       - string:
118           name: MVN
119           default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
120           description: 'Maven selector to be used by shell scripts'
121
122 - scm:
123     name: git-scm
124     scm:
125       - git:
126           credentials-id: 'opendaylight-jenkins-ssh'
127           url: '$GIT_BASE'
128           refspec: ''
129           branches:
130             - 'origin/{branch}'
131           skip-tag: true
132           wipe-workspace: true
133
134 - scm:
135     name: git-scm-with-submodules
136     scm:
137       - git:
138           credentials-id: 'opendaylight-jenkins-ssh'
139           url: '$GIT_BASE'
140           refspec: ''
141           branches:
142             - 'refs/heads/{branch}'
143           skip-tag: true
144           wipe-workspace: true
145           submodule:
146             recursive: true
147
148 - scm:
149     name: gerrit-trigger-scm
150     scm:
151       - git:
152           credentials-id: 'opendaylight-jenkins-ssh'
153           url: '$GIT_BASE'
154           refspec: '{refspec}'
155           branches:
156             - 'origin/$GERRIT_BRANCH'
157           skip-tag: true
158           choosing-strategy: '{choosing-strategy}'
159
160 - wrapper:
161     name: build-timeout
162     wrappers:
163       - timeout:
164           type: absolute
165           timeout: 360
166           fail: true
167
168 # This is a single macro to use for all jobs who vote on every (relevant) patch set.
169 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
170 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
171 # Arguments:
172 #     server: name of gerrit server to listen to
173 #     project: pattern to match triggering projects
174 #     branch: triggering branch name
175 #     files: pattern to match triggering filepaths
176 - trigger:
177     name: gerrit-trigger-patch-submitted
178     triggers:
179       - gerrit:
180           server-name: '{server}'
181           trigger-on:
182             - patchset-created-event:
183                 exclude-drafts: true
184                 exclude-trivial-rebase: false
185                 exclude-no-code-change: false
186             - draft-published-event
187             - comment-added-contains-event:
188                 comment-contains-value: recheck
189           projects:
190             - project-compare-type: ANT
191               project-pattern: '{project}'
192               branches:
193                 - branch-compare-type: ANT
194                   branch-pattern: '**/{branch}'
195               file-paths:
196                 - compare-type: ANT
197                   pattern: '{files}'
198
199 # TODO: Unify argument names across gerrit-trigger-* macros.
200 - trigger:
201     name: gerrit-trigger-patch-merged
202     triggers:
203       - gerrit:
204           server-name: '{server-name}'
205           trigger-on:
206             - change-merged-event
207             - comment-added-contains-event:
208                 comment-contains-value: 'remerge'
209           projects:
210             - project-compare-type: 'ANT'
211               project-pattern: '{name}'
212               branches:
213                 - branch-compare-type: 'ANT'
214                   branch-pattern: '**/{branch}'
215           skip-vote:
216             successful: true
217             failed: true
218             unstable: true
219             notbuilt: true
220           # Force Jenkins always vote the values it should already have voted
221           # during the prior verify phase
222           override-votes: true
223           gerrit-build-started-verified-value: 1
224           gerrit-build-successful-verified-value: 1
225           gerrit-build-failed-verified-value: 1
226           gerrit-build-unstable-verified-value: 1
227           gerrit-build-notbuilt-verified-value: 1
228           gerrit-build-started-codereview-value: 0
229           gerrit-build-successful-codereview-value: 0
230           gerrit-build-failed-codereview-value: 0
231           gerrit-build-unstable-codereview-value: 0
232           gerrit-build-notbuilt-codereview-value: 0
233
234 # TODO: Unify argument names across gerrit-trigger-* macros.
235 - trigger:
236     name: gerrit-trigger-patch-sonar
237     triggers:
238       - gerrit:
239           server-name: '{server-name}'
240           trigger-on:
241             - comment-added-contains-event:
242                 comment-contains-value: 'run-sonar'
243           projects:
244             - project-compare-type: 'ANT'
245               project-pattern: '{name}'
246               branches:
247                 - branch-compare-type: 'ANT'
248                   branch-pattern: '**/master'
249           skip-vote:
250             successful: true
251             failed: true
252             unstable: true
253             notbuilt: true
254
255 - publisher:
256     name: archive-artifacts
257     publishers:
258       - archive:
259           artifacts: '{artifacts}'
260           allow-empty: true
261           fingerprint: true
262           latest-only: true
263
264 - publisher:
265     name: email-notification
266     publishers:
267       - email-ext:
268           recipients: '{email-recipients}'
269           reply-to: ''
270           content-type: default
271           subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
272           body: |
273               $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
274
275               Please refer to the logs server URL for console logs when possible
276               and use the Jenkins Build URL as a last resort.
277
278               Console Logs URL:
279               https://logs.opendaylight.org/$SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
280
281               Jenkins Build URL:
282               $BUILD_URL
283           unstable: true
284           fixed: true
285           send-to:
286             - recipients
287
288 - publisher:
289     name: jacoco-report
290     publishers:
291       - jacoco:
292           exec-pattern: "**/**.exec"
293           class-pattern: "**/classes"
294           source-pattern: "**/src/main/java"
295           exclusion-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**"
296           status-update: true
297           targets:
298             - branch:
299                 healthy: 10
300                 unhealthy: 20
301             - method:
302                 healthy: 50
303                 unhealthy: 40
304
305 - builder:
306     name: opendaylight-infra-stack
307     # opendaylight-infra-stack.sh has a required variable {stack-template} that
308     # must be passed into this macro.
309     builders:
310       - shell: !include-raw: opendaylight-infra-stack.sh
311       - shell: !include-raw-escape: opendaylight-infra-copy-ssh-keys.sh
312
313 - builder:
314     name: opendaylight-infra-push-gerrit-patch
315     # opendaylight-infra-push-gerrit-patch.sh allows a job to push a patch to
316     # Gerrit in an automated fashion. This is meant for tasks that creates
317     # the same patch regularly and needs the ability to detect if an unreviewed
318     # patch already exists. In which case it will update the existing patch.
319     #
320     # Note: This patch assumes the $WORKSPACE contains the project repo with
321     #       the files changed already "git add" and waiting for a "git commit" call.
322     #
323     # This script requires the following JJB variables to be passed in:
324     #     {project}       Gerrit project-name
325     #     {gerrit-topic}  Gerrit topic, please make a unique topic.
326     #     {gerrit-commit-message} Commit message to assign to commit.
327     # NOTE: Requires git review to be installed on node.
328     builders:
329       - shell: !include-raw: opendaylight-infra-push-gerrit-patch.sh
330
331 - builder:
332     name: distribute-build-url
333     # Place URL of the current run of a build job to a file at given path.
334     builders:
335       - shell: 'echo "$BUILD_URL" > {path}/build.url'
336
337 - builder:
338     name: wipe-org-opendaylight-repo
339     builders:
340       - shell: 'if [ -d /tmp/r/org/opendaylight ]; then rm -rf /tmp/r/org/opendaylight; fi'
341
342 - builder:
343     name: wipe-local-maven-repo
344     builders:
345       - shell: 'if [ -d /tmp/r ]; then rm -rf /tmp/r; fi'
346
347 - builder:
348     name: jacoco-nojava-workaround
349     builders:
350       - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
351
352 - builder:
353     name: check-clm
354     builders:
355       - sonatype-clm:
356           application-name: '{application-name}'
357
358 - builder:
359     name: releng-check-unicode
360     builders:
361       - shell: |
362           $WORKSPACE/scripts/check-unicode.sh jjb/
363
364 - builder:
365     name: provide-maven-settings
366     builders:
367       - config-file-provider:
368           files:
369             - file-id: '{global-settings-file}'
370               variable: 'GLOBAL_SETTINGS_FILE'
371             - file-id: '{settings-file}'
372               variable: 'SETTINGS_FILE'
373
374 - builder:
375     name: releng-fetch-p2zip-if-necessary
376     builders:
377       - shell: |
378           # Cleanup any existing zips and metadata before we download the new update site
379           rm -f *.zip *.xml
380       - conditional-step:
381           condition-kind: strings-match
382           condition-string1: '$P2ZIP_URL'
383           condition-string2: ''
384           condition-basedir: workspace
385           steps:
386             # TODO: Figure out latest snapshot version number to pull rather than hardcoding 1.1.1-SNAPSHOT
387             - maven-target:
388                 maven-version: '{maven-version}'
389                 goals: >
390                     org.apache.maven.plugins:maven-dependency-plugin:get
391                     org.apache.maven.plugins:maven-dependency-plugin:copy
392                     -V -B
393                     -Dartifact=org.opendaylight.yangide:org.opendaylight.yangide.update-site:1.1.1-SNAPSHOT:zip
394                     -DoutputDirectory=$WORKSPACE
395                 settings: '{settings}'
396                 global-settings: '{global-settings}'
397
398 - builder:
399     name: releng-generate-p2pom
400     builders:
401       - shell: !include-raw: include-raw-generate-p2pom.sh
402
403 - builder:
404     name: releng-update-p2composite-metadata
405     builders:
406       - shell: !include-raw: include-raw-update-p2composite-metadata.sh
407       - conditional-step:
408           condition-kind: file-exists
409           condition-filename: deploy-composite-repo.xml
410           condition-basedir: workspace
411           steps:
412             - maven-target:
413                 maven-version: '{maven-version}'
414                 pom: 'deploy-composite-repo.xml'
415                 goals: 'clean deploy -V -B -Dmaven.repo.local=/tmp/r'
416                 settings: '{settings}'
417                 global-settings: '{global-settings}'
418
419 - builder:
420     name: releng-stage-release
421     builders:
422       - shell: !include-raw: include-raw-stage-release.sh
423
424 - wrapper:
425     # This wrapper is required for all jobs as it configures the wrappers
426     # needed by OpenDaylight infra.
427     name: opendaylight-infra-wrappers
428     wrappers:
429       - mask-passwords
430       - config-file-provider:
431           files:
432             - file-id: npmrc
433               target: '$HOME/.npmrc'
434             - file-id: rackspace-heat
435               target: '$HOME/.config/openstack/clouds.yaml'
436       - timeout:
437           type: absolute
438           timeout: '{build-timeout}'
439           timeout-var: 'BUILD_TIMEOUT'
440           fail: true
441       - timestamps
442       - ssh-agent-credentials:
443           users:
444             - 'opendaylight-jenkins-ssh'
445       - openstack:
446           single-use: true
447
448 - builder:
449     name: packer-validate
450     builders:
451       - config-file-provider:
452           files:
453             - file-id: 'packer-cloud-env'
454               variable: 'CLOUDENV'
455       - shell: |
456           cd packer
457           varfiles="../packer/vars/*"
458           templates="../packer/templates/*"
459           provision="../packer/provision/*.sh"
460           for v in $varfiles; do
461               [[ "${v##*/}" =~ ^(cloud-env.*)$ ]] && continue
462               for t in $templates; do
463                   export PACKER_LOG="yes" && \
464                   export PACKER_LOG_PATH="packer-validate-${v##*/}-${t##*/}.log" && \
465                               packer.io validate -var-file=$CLOUDENV \
466                               -var-file=$v $t
467                   if [ $? -ne 0 ]; then
468                       break
469                   fi
470               done
471           done
472           for p in $provision; do
473               /bin/bash -n $p > provision-validate-${p##*/}.log 2>&1
474           done
475
476
477 - builder:
478     name: packer-build
479     builders:
480       - config-file-provider:
481           files:
482             - file-id: 'packer-cloud-env'
483               variable: 'CLOUDENV'
484       - shell: |
485           cd packer
486           export PACKER_LOG="yes" && \
487           export PACKER_LOG_PATH="packer-build.log" && \
488                       packer.io build -color=false -var-file=$CLOUDENV \
489                        -var-file=../packer/vars/{platform}.json \
490                        -parallel=false \
491                        ../packer/templates/{template}.json
492
493 - builder:
494     # TODO: Verify signature after downloading users public key from a locally created
495     # repository instead of the public keymesh. This requires a process in place to get ODL
496     # developers public keys into a local repository without increasing the job thoughput.
497     name: verify-gpg-signature
498     builders:
499       - shell: !include-raw: include-raw-verify-gpg-signatures.sh
500
501 - builder:
502     name: opendaylight-infra-jjbini
503     builders:
504       - config-file-provider:
505           files:
506             - file-id: 'jjbini'
507               target: '$HOME/.config/jenkins_jobs/jenkins_jobs.ini'
508
509 - builder:
510     name: distribution-check-wipe
511     # Step zero: Wipe file repositories up front.
512     builders:
513       - shell: |
514           echo "wipe r: the local Maven repository"
515           rm -rfv /tmp/r
516           echo "wipe n: the fake remote (Nexus) repository"
517           rm -rfv /tmp/n
518           echo "wipe t: the transient repository used in some manipulations"
519           rm -rfv /tmp/t
520
521 - builder:
522     name: distribution-check-build-project
523     # Step one: Online build of the project, using local repository /tmp/r/ and deploying artifacts to /tmp/n/.
524     # Ordinary SingleFeatureTest failures are detected in the verify job, so we can use "q" profile here.
525     # Arguments:
526     #   pom: Relative path to pom file to use. Typically '$GERRIT_PROJECT/pom.xml'.
527     builders:
528       - maven-target:
529           maven-version: 'mvn33'
530           pom: '{pom}'
531           goals: |
532               clean deploy dependency:tree
533               -DoutputFile=dependency_tree.txt
534               -Pq
535               -DaltDeploymentRepository=fake-nexus::default::file:///tmp/n/
536               {mvn-opts}
537           java-opts:
538             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
539           settings: 'integration-settings'
540           settings-type: cfp
541           global-settings: 'odl-global-settings'
542           global-settings-type: cfp
543
544 - builder:
545     name: distribution-check-verify-groupid
546     # Step two: Verify all deployed artifacts belong to the project's groupId.
547     # This is done by moving the allowed directories out of /tmp/n and checking no files remained there.
548     # The correct directory is derived from $GERRIT_PROJECT.
549     # TODO: Verify all deployed artifacts are snapshots.
550     # Arguments:
551     #   gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'.
552     builders:
553       - shell: |
554           mkdir -p /tmp/t/org/opendaylight/{gerrit-project}
555           mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/
556           test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false )
557           rm -rf /tmp/n
558           mv /tmp/t /tmp/n
559
560 - builder:
561     name: distribution-check-download-deps
562     # Step three: Online build of integration distribution.
563     # This step is mainly used for downloading other project artifacts.
564     # Running SingleFeaturesTest here does not seem to be required, so -Pq is used again.
565     # Arguments:
566     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
567     builders:
568       - maven-target:
569           maven-version: 'mvn33'
570           pom: '{dist-pom}'
571           goals: |
572               clean install dependency:tree
573               -DoutputFile=dependency_tree.txt
574               -Pq
575               {mvn-opts}
576           java-opts:
577             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
578           settings: 'integration-settings'
579           settings-type: cfp
580           global-settings: 'odl-global-settings'
581           global-settings-type: cfp
582
583 - builder:
584     name: distribution-check-delete-snapshots
585     # Step four: Delete snapshot artifacts from the local repository.
586     # This is critical to detect orphaned artifacts or missing project-internal dependency declarations.
587     # Also other files related to maven repository resolution are removed,
588     # and then empty directories are removed, in order to simplify debugging.
589     builders:
590       - shell: !include-raw-escape: integration-distribution-delete-snaphot-artifacts.sh
591
592 - builder:
593     name: distribution-check-configure-remotes
594     # Now the ugly part. It seems that the only way to tell Maven 2+
595     # which remote repositories to use is via settings.xml file.
596     # So we create such a file here, but it needs most of odlparent:settings.xml
597     builders:
598       - shell: |
599           echo '
600           <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
601             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
602             xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
603             https://maven.apache.org/xsd/settings-1.0.0.xsd">
604             <profiles>
605               <profile>
606                 <id>opendaylight-release</id>
607                 <repositories>
608                   <repository>
609                     <id>opendaylight-mirror</id>
610                     <name>opendaylight</name>
611                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
612                     <releases><updatePolicy>never</updatePolicy></releases>
613                     <snapshots><enabled>false</enabled></snapshots>
614                   </repository>
615                 </repositories>
616                 <pluginRepositories>
617                   <pluginRepository>
618                     <id>opendaylight-plugin-mirror</id>
619                     <name>opendaylight-plugin</name>
620                     <url>https://nexus.opendaylight.org/content/repositories/public/</url>
621                     <releases><updatePolicy>never</updatePolicy></releases>
622                     <snapshots><enabled>false</enabled></snapshots>
623                   </pluginRepository>
624                 </pluginRepositories>
625               </profile>
626               <profile>
627                 <id>file-snapshots</id>
628                 <repositories>
629                   <repository>
630                     <id>file-snapshots</id>
631                     <name>file</name>
632                     <url>file:///tmp/n/</url>
633                     <releases><enabled>false</enabled></releases>
634                   </repository>
635                 </repositories>
636                 <pluginRepositories>
637                   <pluginRepository>
638                     <id>file-plugin-snapshots</id>
639                     <name>file-plugin</name>
640                     <url>file:///tmp/n/</url>
641                     <releases><enabled>false</enabled></releases>
642                   </pluginRepository>
643                 </pluginRepositories>
644               </profile>
645             </profiles>
646             <activeProfiles>
647               <activeProfile>file-snapshots</activeProfile>
648               <activeProfile>opendaylight-release</activeProfile>
649             </activeProfiles>
650           </settings>
651           ' > fake_remotes.xml
652           # Notes: The settings are minimal in order to detect breakage scenarios,
653           # while allowing for the following quirks:
654           # * Some plugins seem to have hardcoded repos, for example check-license looks at repository.apache.org
655           # * Some plugin artifacts (related to surefire) are not downloaded when tests are skipped.
656           # * populate-local-repo looks at oss.sonatype.org and does not store things (like guava) to /tmp/r
657
658 - builder:
659     name: distribution-check-repeat-build
660     # Step five: Repeat the distribution build but with the new settings.
661     # Here, only the project snapshot artifacts deployed to /tmp/n are available,
662     # which faithfully reproduces conditions in later verify-like job runs.
663     # We cannot use --offline, because: "Cannot access file (file:///tmp/n) in offline mode"
664     # This is where SingleFeatureTest is not skipped.
665     # Arguments:
666     #   dist-pom: Relative path to pom file to use. 'distribution/pom.xml' is recommended.
667     builders:
668       - maven-target:
669           maven-version: 'mvn33'
670           pom: '{dist-pom}'
671           goals: |
672               clean install dependency:tree
673               -DoutputFile=dependency_tree.txt -s fake_remotes.xml
674               -Pq
675               -DskipTests=false
676               {mvn-opts}
677           java-opts:
678             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'