Merge "Consolidate controller parameters into new macro"
[releng/builder.git] / jjb / integration / integration-macros.yaml
1 ---
2 # TODO: Make all bash constants more readable.
3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
4
5 - parameter:
6     name: integration-csit-parameters
7     parameters:
8       - opendaylight-infra-parameters:
9           os-cloud: '{os-cloud}'
10           project: 'integration/test'
11           branch: '{branch}'
12           refspec: 'refs/heads/{branch}'
13           artifacts: '{archive-artifacts}'
14       # CSIT
15       - integration-bundle-url:
16           bundle-url: '{bundle-url}'
17       - integration-repo-url:
18           repo-url: '{repo-url}'
19       - integration-test-options:
20           test-options: '{robot-options}'
21       - integration-controller-parameters:
22           controller-debug-map: '{debug-map}'
23           controller-features: '{install-features}'
24           controller-scope: '{controller-scope}'
25           controller-use-features-boot: '{use-features-boot}'
26
27 - parameter:
28     name: integration-controller-parameters
29     parameters:
30       - string:
31           name: CONTROLLERSCOPE
32           default: '{controller-scope}'
33           description: 'Feature Only or Feature All test'
34
35       - string:
36           name: CONTROLLERFEATURES
37           default: '{controller-features}'
38           description: 'Features to install in the controller separated by comma'
39
40       - string:
41           name: CONTROLLERDEBUGMAP
42           default: '{controller-debug-map}'
43           description: |
44               Key:Value map of debugs to enable on the controller separated by spaces.
45               The module should not include org.opendaylight.
46               The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
47               (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
48
49       - string:
50           name: USEFEATURESBOOT
51           default: '{controller-use-features-boot}'
52           description: 'Boolean to toggle whether featuresBoot is appended or ignored'
53
54
55 - parameter:
56     name: integration-distribution-branch
57     parameters:
58       - string:
59           name: DISTROBRANCH
60           default: '{branch}'
61           description: 'Distribution GIT branch'
62
63 - parameter:
64     name: distribution-karaf-version
65     # Arguments:
66     #   karaf-version: Provide, otherwise defaults.yaml value is used.
67     parameters:
68       - string:
69           name: KARAF_VERSION
70           default: '{karaf-version}'
71           description: 'karaf3, karaf4, odl, affects ODL deploy scripts'
72
73 - parameter:
74     name: integration-bundle-url
75     parameters:
76       - string:
77           name: BUNDLE_URL
78           default: '{bundle-url}'
79           description: 'URL to karaf distribution zip'
80
81 - parameter:
82     name: integration-repo-url
83     parameters:
84       - string:
85           name: REPO_URL
86           default: '{repo-url}'
87           description: 'URL to feature repo artifact'
88
89 - parameter:
90     name: integration-jdk-version
91     parameters:
92       - string:
93           name: JDKVERSION
94           default: '{jdkversion}'
95           description: 'Parameter to indicate JAVA Version'
96
97 - parameter:
98     name: integration-distribution-stream
99     parameters:
100       - string:
101           name: DISTROSTREAM
102           default: '{stream}'
103           description: 'Distribution stream string, for suites to know which behavior to expect'
104
105 - parameter:
106     name: integration-script-plan
107     parameters:
108       - string:
109           name: SCRIPTPLAN
110           default: '{script-plan}'
111           description: 'Script plan we will run before downloading and un-archiving am ODL distribution'
112
113 - parameter:
114     name: integration-config-plan
115     parameters:
116       - string:
117           name: CONFIGPLAN
118           default: '{config-plan}'
119           description: 'Config plan we will run after un-archiving and starting an ODL distribution'
120
121 - parameter:
122     name: integration-stream-test-plan
123     parameters:
124       - string:
125           name: STREAMTESTPLAN
126           default: '{stream-test-plan}'
127           description: 'Stream-specific test plan we will run'
128
129 - parameter:
130     name: integration-test-plan
131     parameters:
132       - string:
133           name: TESTPLAN
134           default: '{test-plan}'
135           description: 'General test plan we will run unless stream-specific one is found'
136
137 - parameter:
138     name: integration-test-suites
139     parameters:
140       - string:
141           name: SUITES
142           default: '{test-suites}'
143           description: |
144               List of space-separated suites. Useful when running specific suites within a testplan.
145               Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
146               (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
147                openstack/tempest/tempest.robot netvirt/elan/elan.robot).
148
149 - parameter:
150     name: integration-test-options
151     parameters:
152       - string:
153           name: TESTOPTIONS
154           default: '{test-options}'
155           description: 'Robot command options'
156
157 - parameter:
158     name: integration-distribution-git-url
159     parameters:
160       - string:
161           name: DISTROGITURL
162           default: '{git-url}/integration/distribution'
163           description: 'Distribution GIT URL (do not modify)'
164
165 - parameter:
166     name: integration-patch-refspec
167     parameters:
168       - string:
169           name: PATCHREFSPEC
170           default: '{branch}'
171           description: 'Integration Patch Refspec'
172
173 # Macro: integration-gerrit-scm
174 # Operation: this macro downloads a project gerrit
175 # Used by: all csit jobs
176
177 - scm:
178     name: integration-gerrit-scm
179     scm:
180       - git:
181           credentials-id: 'opendaylight-jenkins-ssh'
182           url: '$GIT_BASE'
183           basedir: '{basedir}'
184           refspec: '{refspec}'
185           branches:
186             - 'origin/{branch}'
187           skip-tag: true
188           choosing-strategy: 'gerrit'
189
190 # Macro: integration-distribution-scm
191 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
192 # Used by: integration-patch-test-{stream} job template
193
194 - scm:
195     name: integration-distribution-scm
196     scm:
197       - git:
198           credentials-id: 'opendaylight-jenkins-ssh'
199           basedir: 'distribution'
200           url: '$DISTROGITURL'
201           refspec: ''
202           branches:
203             - 'origin/{branch}'
204           skip-tag: true
205
206 # Macro: integration-trigger-patch-submitted
207 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
208 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
209 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
210 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
211 # Arguments:
212 #     gerrit-server-name: name of gerrit server to listen to
213 #     project: pattern to match triggering projects
214 #     branch: triggering branch name
215 #     files: pattern to match triggering filepaths
216 - trigger:
217     name: integration-trigger-patch-submitted
218     triggers:
219       - gerrit:
220           server-name: '{gerrit-server-name}'
221           trigger-on:
222             - patchset-created-event:
223                 exclude-drafts: true
224                 exclude-trivial-rebase: false
225                 exclude-no-code-change: false
226             - draft-published-event
227             - comment-added-contains-event:
228                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
229           override-votes: true
230           gerrit-build-unstable-verified-value: +1
231           gerrit-build-unstable-codereview-value: 0
232           projects:
233             - project-compare-type: ANT
234               project-pattern: '{project}'
235               branches:
236                 - branch-compare-type: ANT
237                   branch-pattern: '**/{branch}'
238               file-paths:
239                 - compare-type: ANT
240                   pattern: '{files}'
241
242 # Macro: integration-csit-only
243 # Operation: this macro sets a trigger on reverse job list
244 # Used by: csit -only- job templates
245
246 - trigger:
247     name: integration-csit-trigger
248     triggers:
249       - reverse:
250           jobs: '{jobs}'
251           result: 'success'
252
253 # Macro: integration-openstack-controller-mininet
254 # Operation: this macro will spin the controller and mininet vms
255 # Used by: {project}-csit-* job templates
256
257 - wrapper:
258     name: integration-openstack-controller-mininet
259     wrappers:
260       - openstack:
261           instances:
262             - cloud-name: 'ODLRPC'
263               template-name: '{controller-image}'
264               count: '{controller-vms}'
265             - cloud-name: 'ODLRPC'
266               template-name: '{mininet-image}'
267               count: '{mininet-vms}'
268
269 # Macro: integration-openstack-controller-devstack
270 # Operation: this macro will spin the controller and devstack vms
271 # Used by: {project}-csit-openstack job templates
272 - wrapper:
273     name: integration-openstack-controller-devstack
274     wrappers:
275       - openstack:
276           instances:
277             - cloud-name: 'ODLRPC'
278               template-name: '{controller-image}'
279               count: '{controller-vms}'
280             - cloud-name: 'ODLRPC'
281               template-name: '{openstack-image}'
282               count: '{openstack-vms}'
283
284 #################
285 # Shell Scripts #
286 #################
287
288 # Macro: integration-infra-stack
289 # Operation: Sets environment and then calls opendaylight-infra-stack to spin
290 #            up csit lab using openstack-heat.
291 # Used by: {project}-csit-* job templates
292
293 - builder:
294     name: integration-infra-stack
295     builders:
296       - integration-cleanup-workspace
297       - shell: |
298           #!/bin/bash
299           # Setup openstack envirnoment file for use by
300           # the opendaylight-infra-stack macro
301           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
302           parameters:
303               vm_0_count: {vm_0_count}
304               vm_0_flavor: {vm_0_flavor}
305               vm_0_image: {vm_0_image}
306               vm_1_count: {vm_1_count}
307               vm_1_flavor: {vm_1_flavor}
308               vm_1_image: {vm_1_image}
309           EOF
310           echo "Contents of opendaylight-infra-environment.yaml ..."
311           cat $WORKSPACE/opendaylight-infra-environment.yaml
312       - opendaylight-infra-stack:
313           stack-template: '{stack-template}'
314
315 # Macro: integration-infra-stack-3-type
316 # Operation: Sets environment and then calls opendaylight-infra-stack to spin
317 #            up csit lab using openstack-heat.
318 # Used by: {project}-csit-* job templates
319
320 - builder:
321     name: integration-infra-stack-3-type
322     builders:
323       - integration-cleanup-workspace
324       - shell: |
325           #!/bin/bash
326           # Setup openstack envirnoment file for use by
327           # the opendaylight-infra-stack macro
328           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
329           parameters:
330               vm_0_count: {vm_0_count}
331               vm_0_flavor: {vm_0_flavor}
332               vm_0_image: {vm_0_image}
333               vm_1_count: {vm_1_count}
334               vm_1_flavor: {vm_1_flavor}
335               vm_1_image: {vm_1_image}
336               vm_2_count: {vm_2_count}
337               vm_2_flavor: {vm_2_flavor}
338               vm_2_image: {vm_2_image}
339           EOF
340           echo "Contents of opendaylight-infra-environment.yaml ..."
341           cat $WORKSPACE/opendaylight-infra-environment.yaml
342       - opendaylight-infra-stack:
343           stack-template: 'csit-3-instance-type.yaml'
344
345 # Macro: integration-apex-stack-3-type
346 # Operation: Sets environment and then calls opendaylight-apex-stack to spin
347 #            up csit lab using openstack-heat.
348 # Used by: {project}-csit-* apex related job templates
349
350 - builder:
351     name: integration-apex-stack-3-type
352     builders:
353       - integration-cleanup-workspace
354       - shell: |
355           #!/bin/bash
356           # Setup openstack envirnoment file for use by
357           # the opendaylight-infra-stack macro
358           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
359           parameters:
360               vm_0_count: {vm_0_count}
361               vm_0_flavor: {vm_0_flavor}
362               vm_0_image: {vm_0_image}
363               vm_1_count: {vm_1_count}
364               vm_1_flavor: {vm_1_flavor}
365               vm_1_image: {vm_1_image}
366               vm_2_count: {vm_2_count}
367               vm_2_flavor: {vm_2_flavor}
368               vm_2_image: {vm_2_image}
369           EOF
370           echo "Contents of opendaylight-infra-environment.yaml ..."
371           cat $WORKSPACE/opendaylight-infra-environment.yaml
372       - opendaylight-apex-stack:
373           stack-template: 'csit-apex-3-instance-type.yaml'
374
375 # Macro: integration-infra-stack-4-type
376 # Operation: Sets environment and then calls opendaylight-infra-stack to spin
377 #            up csit lab using openstack-heat.
378 # Used by: {project}-csit-* job templates
379
380 - builder:
381     name: integration-infra-stack-4-type
382     builders:
383       - integration-cleanup-workspace
384       - shell: |
385           #!/bin/bash
386           # Setup openstack envirnoment file for use by
387           # the opendaylight-infra-stack macro
388           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
389           parameters:
390               vm_0_count: {vm_0_count}
391               vm_0_flavor: {vm_0_flavor}
392               vm_0_image: {vm_0_image}
393               vm_1_count: {vm_1_count}
394               vm_1_flavor: {vm_1_flavor}
395               vm_1_image: {vm_1_image}
396               vm_2_count: {vm_2_count}
397               vm_2_flavor: {vm_2_flavor}
398               vm_2_image: {vm_2_image}
399               vm_3_count: {vm_3_count}
400               vm_3_flavor: {vm_3_flavor}
401               vm_3_image: {vm_3_image}
402           EOF
403           echo "Contents of opendaylight-infra-environment.yaml ..."
404           cat $WORKSPACE/opendaylight-infra-environment.yaml
405       - opendaylight-infra-stack:
406           stack-template: 'csit-4-instance-type.yaml'
407
408 # Macro: integration-get-apex-addresses
409 # Operation: this macro gets the IP addresses of the dynamic vms from apex snapshots
410 # Used by: {project}-csit-* apex related job templates
411
412 - builder:
413     name: integration-get-apex-addresses
414     builders:
415       - shell: !include-raw: integration-get-apex-addresses.sh
416
417 # Macro: integration-apex-run-tests
418 # Operation: this macro just runs the robot suites
419 # Used by: {project}-csit-* apex related job templates
420
421 - builder:
422     name: integration-apex-run-tests
423     builders:
424       - shell: !include-raw: integration-apex-run-tests.sh
425
426 # Macro: integration-get-slave-addresses
427 # Operation: this macro gets the IP addresses of the dynamic vms
428 # Used by: {project}-csit-* job templates
429
430 - builder:
431     name: integration-get-slave-addresses
432     builders:
433       - shell: !include-raw: integration-get-slave-addresses.sh
434
435 - builder:
436     name: integration-set-variables
437     # Operation: Construct variables not requiring online detection and export using set_variables.env
438     # Required bash variables:
439     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
440     builders:
441       - shell: !include-raw: integration-set-variables.sh
442       - inject:
443           properties-file: set_variables.env
444
445 - builder:
446     name: integration-detect-variables
447     # Operation: Construct variables including online detection and export using detect_variables.env
448     #   This calls integration-set-variables internally.
449     # Required bash variables:
450     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
451     builders:
452       - integration-set-variables
453       - shell: !include-raw: integration-detect-variables.sh
454       - inject:
455           properties-file: detect_variables.env
456
457 # Macro: integration-deply-controller-run-test
458 # Operation: this macro deploys single contoller and runs test
459 # Used by: {project}-csit-1node-* job templates
460
461 - builder:
462     name: integration-deploy-controller-run-test
463     builders:
464       - shell: !include-raw: integration-deploy-controller-run-test.sh
465
466 # Macro: integration-configure-clustering
467 # Operation: this macro configures the clustering
468 # Used by: {project}-csit-3node-* job templates
469
470 - builder:
471     name: integration-configure-clustering
472     builders:
473       - shell: !include-raw: integration-configure-clustering.sh
474
475 # Macro: integration-start-cluster-run-test
476 # Operation: this macro starts the 3-node cluster and runs test
477 # Used by: {project}-csit-3node-* job templates
478
479 - builder:
480     name: integration-start-cluster-run-test
481     builders:
482       - shell: !include-raw: integration-start-cluster-run-test.sh
483
484 # Macro: integration-install-robotframework
485 # Operation: Installs robotframework using pip to a virtualenv
486 # Used by: {project}-csit-* job templates
487
488 - builder:
489     name: integration-install-robotframework
490     builders:
491       - shell: !include-raw: integration-install-robotframework.sh
492
493 # Macro: integration-cleanup-tmp
494 # Operation: Cleans up temporary files created by build
495 # Used by: {project}-csit-* job templates
496
497 - builder:
498     name: integration-cleanup-tmp
499     builders:
500       - shell: !include-raw: integration-cleanup-tmp.sh
501
502 # Macro: integration-multipatch-builder
503 # Operation: checks out multiple patches and builds custom distribution
504 # Used by: integration-multipatch-test-* job templates
505
506 - builder:
507     name: integration-multipatch-builder
508     builders:
509       - shell: !include-raw: multipatch-distribution.sh
510
511 # Macro: integration-cleanup-workspace
512 # Operation: Cleans up files possibly left there by the previous build
513 # Used by: {project}-csit-* job templates
514
515 - builder:
516     name: integration-cleanup-workspace
517     builders:
518       - shell: !include-raw: integration-cleanup-workspace.sh
519
520 # Macro: integration-rebase-patch
521 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
522 # Used by: integration-patch-distribution-* job template
523
524 - builder:
525     name: integration-rebase-gerrit-patch
526     builders:
527       - shell: !include-raw: integration-rebase-gerrit-patch.sh
528
529 # Macro: integration-install-common-functions
530 # Operation: Copy the common-functions.sh script to csit nodes
531 # Used by: {project}-csit-* job templates
532 - builder:
533     name: integration-install-common-functions
534     builders:
535       - shell: !include-raw:
536           - copy-common-functions.sh
537           - common-functions.sh
538
539 - builder:
540     name: integration-deploy-openstack-run-test
541     builders:
542       - shell: !include-raw:
543           - integration-deploy-openstack-run-test.sh
544
545 - builder:
546     name: integration-apex-image-manager
547     builders:
548       - shell: !include-raw:
549           - ../opendaylight-infra-apex-image-management.sh
550
551 - builder:
552     name: integration-compare-distributions
553     # Operation: will compare an already created distribtion (probably last
554     # published to nexus) to the current distribution being created.
555     # The comparison is done to find the list of patches that are new to the
556     # current distribution.
557     # Required bash variables:
558     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
559     builders:
560       - shell: !include-raw: integration-compare-distributions.sh
561
562 - builder:
563     name: integration-upload-distribution
564     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
565     # Arguments:
566     #   {dist-pom} path to pom file containing the version
567     # Required bash variables:
568     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
569     #     Use distribution-karaf-version macro to customize.
570     builders:
571       - config-file-provider:
572           files:
573             - file-id: global-settings
574               variable: GLOBAL_SETTINGS_FILE
575             - file-id: integration-settings
576               variable: SETTINGS_FILE
577       - integration-set-variables
578       - inject:
579           properties-content: BUNDLE_POM={dist-pom}
580       - shell: !include-raw-escape: integration-upload-distribution.sh
581       - inject:
582           properties-file: integration-upload-distribution.env
583
584 ##############
585 # Publishers #
586 ##############
587
588 - publisher:
589     name: integration-csit-publish
590     publishers:
591       - robot:
592           output-path: ''
593           other-files: '{obj:robot_other_files}'
594           unstable-threshold: '{robot-unstable-threshold}'
595           pass-threshold: '{robot-pass-threshold}'
596           only-critical: false
597       - integration-csit-archive-build
598       - integration-csit-run-postscript
599       - releng-openstack-stack-delete
600
601 - publisher:
602     name: integration-csit-archive-build
603     publishers:
604       - postbuildscript:
605           builders:
606             - role: BOTH
607               build-on:
608                 - ABORTED
609                 - FAILURE
610                 - NOT_BUILT
611                 - SUCCESS
612                 - UNSTABLE
613               build-steps:
614                 - shell: |
615                     #!/bin/bash
616                     echo "Archiving csit artifacts"
617                     cd $WORKSPACE
618                     mkdir -p ./archives
619                     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
620                         NODE_FOLDER="./archives/odl_${i}"
621                         mkdir -p ${NODE_FOLDER}
622                         mv odl${i}_* ${NODE_FOLDER} || true
623                         mv karaf_${i}_*_threads* ${NODE_FOLDER} || true
624                         mv *_${i}.png ${NODE_FOLDER} || true
625                         mv /tmp/odl${i}_* ${NODE_FOLDER} || true
626                     done
627                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
628                     unzip -d ./archives robot-plugin.zip
629                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
630                     # TODO: Can the following line ever fail?
631                     find . -type f -name *.hprof -print0 \
632                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
633                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
634           mark-unstable-if-failed: true
635
636
637 - publisher:
638     name: integration-csit-run-postscript
639     publishers:
640       - postbuildscript:
641           builders:
642             - role: BOTH
643               build-on:
644                 - ABORTED
645                 - FAILURE
646                 - NOT_BUILT
647                 - SUCCESS
648                 - UNSTABLE
649               build-steps:
650                 - config-file-provider:
651                     files:
652                       - file-id: 'odl-elastic-cloud'
653                         target: '${HOME}/.netrc'
654                 - shell: |
655                     #!/bin/bash
656                     #The following script runs any configurable postplan stored in test/csit/postplans.
657                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
658                         echo "postplan exists!!!"
659                         echo "Changing the postplan path..."
660                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
661                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
662                         cat postplan.txt
663                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
664                             echo "Executing ${line}..."
665                             ${line}
666                         done
667                     fi
668                     rm ${HOME}/.netrc
669           mark-unstable-if-failed: true
670
671 - publisher:
672     name: integration-csit-collect-netvirt-logs
673     publishers:
674       - postbuildscript:
675           builders:
676             - role: BOTH
677               build-on:
678                 - ABORTED
679                 - FAILURE
680                 - NOT_BUILT
681                 - SUCCESS
682                 - UNSTABLE
683               build-steps:
684                 - inject:
685                     properties-file: 'slave_addresses.txt'
686                 - shell: |
687                     #!/bin/bash
688                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
689                     collect_netvirt_logs
690           mark-unstable-if-failed: true