jjb/integration/integration-macros.yaml

   1 ---
   2 # TODO: Make all bash constants more readable.
   3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
   4 - parameter:
   5     name: integration-distribution-branch
   6     parameters:
   7       - string:
   8           name: DISTROBRANCH
   9           default: '{branch}'
  10           description: 'Distribution GIT branch'
  11
  12 - parameter:
  13     name: distribution-karaf-version
  14     # Arguments:
  15     #   karaf-version: Provide, otherwise defaults.yaml value is used.
  16     parameters:
  17       - string:
  18           name: KARAF_VERSION
  19           default: '{karaf-version}'
  20           description: 'karaf3, karaf4, odl, affects ODL deploy scripts'
  21
  22 - parameter:
  23     name: integration-bundle-url
  24     parameters:
  25       - string:
  26           name: BUNDLE_URL
  27           default: '{bundle-url}'
  28           description: 'URL to karaf distribution zip'
  29
  30 - parameter:
  31     name: integration-repo-url
  32     parameters:
  33       - string:
  34           name: REPO_URL
  35           default: '{repo-url}'
  36           description: 'URL to feature repo artifact'
  37
  38 - parameter:
  39     name: integration-jdk-version
  40     parameters:
  41       - string:
  42           name: JDKVERSION
  43           default: '{jdkversion}'
  44           description: 'Parameter to indicate JAVA Version'
  45
  46 - parameter:
  47     name: integration-controller-scope
  48     parameters:
  49       - string:
  50           name: CONTROLLERSCOPE
  51           default: '{controller-scope}'
  52           description: 'Feature Only or Feature All test'
  53
  54 - parameter:
  55     name: integration-controller-features
  56     parameters:
  57       - string:
  58           name: CONTROLLERFEATURES
  59           default: '{controller-features}'
  60           description: 'Features to install in the controller separated by comma'
  61
  62 - parameter:
  63     name: integration-controller-debug-map
  64     parameters:
  65       - string:
  66           name: CONTROLLERDEBUGMAP
  67           default: '{controller-debug-map}'
  68           description: |
  69               Key:Value map of debugs to enable on the controller separated by spaces.
  70               The module should not include org.opendaylight.
  71               The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
  72               (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
  73
  74 - parameter:
  75     name: integration-distribution-stream
  76     parameters:
  77       - string:
  78           name: DISTROSTREAM
  79           default: '{stream}'
  80           description: 'Distribution stream string, for suites to know which behavior to expect'
  81
  82 - parameter:
  83     name: integration-script-plan
  84     parameters:
  85       - string:
  86           name: SCRIPTPLAN
  87           default: '{script-plan}'
  88           description: 'Script plan we will run before downloading and un-archiving am ODL distribution'
  89
  90 - parameter:
  91     name: integration-config-plan
  92     parameters:
  93       - string:
  94           name: CONFIGPLAN
  95           default: '{config-plan}'
  96           description: 'Config plan we will run after un-archiving and starting an ODL distribution'
  97
  98 - parameter:
  99     name: integration-stream-test-plan
 100     parameters:
 101       - string:
 102           name: STREAMTESTPLAN
 103           default: '{stream-test-plan}'
 104           description: 'Stream-specific test plan we will run'
 105
 106 - parameter:
 107     name: integration-test-plan
 108     parameters:
 109       - string:
 110           name: TESTPLAN
 111           default: '{test-plan}'
 112           description: 'General test plan we will run unless stream-specific one is found'
 113
 114 - parameter:
 115     name: integration-test-suites
 116     parameters:
 117       - string:
 118           name: SUITES
 119           default: '{test-suites}'
 120           description: |
 121               List of space-separated suites. Useful when running specific suites within a testplan.
 122               Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
 123               (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
 124                openstack/tempest/tempest.robot netvirt/elan/elan.robot).
 125
 126 - parameter:
 127     name: integration-test-options
 128     parameters:
 129       - string:
 130           name: TESTOPTIONS
 131           default: '{test-options}'
 132           description: 'Robot command options'
 133
 134 - parameter:
 135     name: integration-distribution-git-url
 136     parameters:
 137       - string:
 138           name: DISTROGITURL
 139           default: '{git-url}/integration/distribution'
 140           description: 'Distribution GIT URL (do not modify)'
 141
 142 - parameter:
 143     name: integration-patch-refspec
 144     parameters:
 145       - string:
 146           name: PATCHREFSPEC
 147           default: '{branch}'
 148           description: 'Integration Patch Refspec'
 149
 150 - parameter:
 151     name: integration-use-features-boot
 152     parameters:
 153       - string:
 154           name: USEFEATURESBOOT
 155           default: '{use-features-boot}'
 156           description: 'Boolean to toggle whether featuresBoot is appended or ignored'
 157
 158 # Macro: integration-gerrit-scm
 159 # Operation: this macro downloads a project gerrit
 160 # Used by: all csit jobs
 161
 162 - scm:
 163     name: integration-gerrit-scm
 164     scm:
 165       - git:
 166           credentials-id: 'opendaylight-jenkins-ssh'
 167           url: '$GIT_BASE'
 168           basedir: '{basedir}'
 169           refspec: '{refspec}'
 170           branches:
 171             - 'origin/{branch}'
 172           skip-tag: true
 173           choosing-strategy: 'gerrit'
 174
 175 # Macro: integration-distribution-scm
 176 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
 177 # Used by: integration-patch-test-{stream} job template
 178
 179 - scm:
 180     name: integration-distribution-scm
 181     scm:
 182       - git:
 183           credentials-id: 'opendaylight-jenkins-ssh'
 184           basedir: 'distribution'
 185           url: '$DISTROGITURL'
 186           refspec: ''
 187           branches:
 188             - 'origin/{branch}'
 189           skip-tag: true
 190
 191 # Macro: integration-trigger-patch-submitted
 192 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
 193 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
 194 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
 195 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
 196 # Arguments:
 197 #     gerrit-server-name: name of gerrit server to listen to
 198 #     project: pattern to match triggering projects
 199 #     branch: triggering branch name
 200 #     files: pattern to match triggering filepaths
 201 - trigger:
 202     name: integration-trigger-patch-submitted
 203     triggers:
 204       - gerrit:
 205           server-name: '{gerrit-server-name}'
 206           trigger-on:
 207             - patchset-created-event:
 208                 exclude-drafts: true
 209                 exclude-trivial-rebase: false
 210                 exclude-no-code-change: false
 211             - draft-published-event
 212             - comment-added-contains-event:
 213                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 214           override-votes: true
 215           gerrit-build-unstable-verified-value: +1
 216           gerrit-build-unstable-codereview-value: 0
 217           projects:
 218             - project-compare-type: ANT
 219               project-pattern: '{project}'
 220               branches:
 221                 - branch-compare-type: ANT
 222                   branch-pattern: '**/{branch}'
 223               file-paths:
 224                 - compare-type: ANT
 225                   pattern: '{files}'
 226
 227 # Macro: integration-csit-only
 228 # Operation: this macro sets a trigger on reverse job list
 229 # Used by: csit -only- job templates
 230
 231 - trigger:
 232     name: integration-csit-trigger
 233     triggers:
 234       - reverse:
 235           jobs: '{jobs}'
 236           result: 'success'
 237
 238 # Macro: integration-openstack-controller-mininet
 239 # Operation: this macro will spin the controller and mininet vms
 240 # Used by: {project}-csit-* job templates
 241
 242 - wrapper:
 243     name: integration-openstack-controller-mininet
 244     wrappers:
 245       - openstack:
 246           instances:
 247             - cloud-name: 'ODLRPC'
 248               template-name: '{controller-image}'
 249               count: '{controller-vms}'
 250             - cloud-name: 'ODLRPC'
 251               template-name: '{mininet-image}'
 252               count: '{mininet-vms}'
 253
 254 # Macro: integration-openstack-controller-devstack
 255 # Operation: this macro will spin the controller and devstack vms
 256 # Used by: {project}-csit-openstack job templates
 257 - wrapper:
 258     name: integration-openstack-controller-devstack
 259     wrappers:
 260       - openstack:
 261           instances:
 262             - cloud-name: 'ODLRPC'
 263               template-name: '{controller-image}'
 264               count: '{controller-vms}'
 265             - cloud-name: 'ODLRPC'
 266               template-name: '{openstack-image}'
 267               count: '{openstack-vms}'
 268
 269 #################
 270 # Shell Scripts #
 271 #################
 272
 273 # Macro: integration-infra-stack
 274 # Operation: Sets environment and then calls opendaylight-infra-stack to spin
 275 #            up csit lab using openstack-heat.
 276 # Used by: {project}-csit-* job templates
 277
 278 - builder:
 279     name: integration-infra-stack
 280     builders:
 281       - integration-cleanup-workspace
 282       - shell: |
 283           #!/bin/bash
 284           # Setup openstack envirnoment file for use by
 285           # the opendaylight-infra-stack macro
 286           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
 287           parameters:
 288               vm_0_count: {vm_0_count}
 289               vm_0_flavor: {vm_0_flavor}
 290               vm_0_image: {vm_0_image}
 291               vm_1_count: {vm_1_count}
 292               vm_1_flavor: {vm_1_flavor}
 293               vm_1_image: {vm_1_image}
 294           EOF
 295           echo "Contents of opendaylight-infra-environment.yaml ..."
 296           cat $WORKSPACE/opendaylight-infra-environment.yaml
 297       - opendaylight-infra-stack:
 298           stack-template: '{stack-template}'
 299
 300 # Macro: integration-infra-stack-3-type
 301 # Operation: Sets environment and then calls opendaylight-infra-stack to spin
 302 #            up csit lab using openstack-heat.
 303 # Used by: {project}-csit-* job templates
 304
 305 - builder:
 306     name: integration-infra-stack-3-type
 307     builders:
 308       - integration-cleanup-workspace
 309       - shell: |
 310           #!/bin/bash
 311           # Setup openstack envirnoment file for use by
 312           # the opendaylight-infra-stack macro
 313           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
 314           parameters:
 315               vm_0_count: {vm_0_count}
 316               vm_0_flavor: {vm_0_flavor}
 317               vm_0_image: {vm_0_image}
 318               vm_1_count: {vm_1_count}
 319               vm_1_flavor: {vm_1_flavor}
 320               vm_1_image: {vm_1_image}
 321               vm_2_count: {vm_2_count}
 322               vm_2_flavor: {vm_2_flavor}
 323               vm_2_image: {vm_2_image}
 324           EOF
 325           echo "Contents of opendaylight-infra-environment.yaml ..."
 326           cat $WORKSPACE/opendaylight-infra-environment.yaml
 327       - opendaylight-infra-stack:
 328           stack-template: 'csit-3-instance-type.yaml'
 329
 330 # Macro: integration-infra-stack-4-type
 331 # Operation: Sets environment and then calls opendaylight-infra-stack to spin
 332 #            up csit lab using openstack-heat.
 333 # Used by: {project}-csit-* job templates
 334
 335 - builder:
 336     name: integration-infra-stack-4-type
 337     builders:
 338       - integration-cleanup-workspace
 339       - shell: |
 340           #!/bin/bash
 341           # Setup openstack envirnoment file for use by
 342           # the opendaylight-infra-stack macro
 343           cat > $WORKSPACE/opendaylight-infra-environment.yaml << EOF
 344           parameters:
 345               vm_0_count: {vm_0_count}
 346               vm_0_flavor: {vm_0_flavor}
 347               vm_0_image: {vm_0_image}
 348               vm_1_count: {vm_1_count}
 349               vm_1_flavor: {vm_1_flavor}
 350               vm_1_image: {vm_1_image}
 351               vm_2_count: {vm_2_count}
 352               vm_2_flavor: {vm_2_flavor}
 353               vm_2_image: {vm_2_image}
 354               vm_3_count: {vm_3_count}
 355               vm_3_flavor: {vm_3_flavor}
 356               vm_3_image: {vm_3_image}
 357           EOF
 358           echo "Contents of opendaylight-infra-environment.yaml ..."
 359           cat $WORKSPACE/opendaylight-infra-environment.yaml
 360       - opendaylight-infra-stack:
 361           stack-template: 'csit-4-instance-type.yaml'
 362
 363 # Macro: integration-get-slave-addresses
 364 # Operation: this macro gets the IP addresses of the dynamic vms
 365 # Used by: {project}-csit-* job templates
 366
 367 - builder:
 368     name: integration-get-slave-addresses
 369     builders:
 370       - shell: !include-raw: integration-get-slave-addresses.sh
 371
 372 - builder:
 373     name: integration-set-variables
 374     # Operation: Construct variables not requiring online detection and export using set_variables.env
 375     # Required bash variables:
 376     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 377     builders:
 378       - shell: !include-raw: integration-set-variables.sh
 379       - inject:
 380           properties-file: set_variables.env
 381
 382 - builder:
 383     name: integration-detect-variables
 384     # Operation: Construct variables including online detection and export using detect_variables.env
 385     #   This calls integration-set-variables internally.
 386     # Required bash variables:
 387     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 388     builders:
 389       - integration-set-variables
 390       - shell: !include-raw: integration-detect-variables.sh
 391       - inject:
 392           properties-file: detect_variables.env
 393
 394 # Macro: integration-deply-controller-run-test
 395 # Operation: this macro deploys single contoller and runs test
 396 # Used by: {project}-csit-1node-* job templates
 397
 398 - builder:
 399     name: integration-deploy-controller-run-test
 400     builders:
 401       - shell: !include-raw: integration-deploy-controller-run-test.sh
 402
 403 # Macro: integration-configure-clustering
 404 # Operation: this macro configures the clustering
 405 # Used by: {project}-csit-3node-* job templates
 406
 407 - builder:
 408     name: integration-configure-clustering
 409     builders:
 410       - shell: !include-raw: integration-configure-clustering.sh
 411
 412 # Macro: integration-start-cluster-run-test
 413 # Operation: this macro starts the 3-node cluster and runs test
 414 # Used by: {project}-csit-3node-* job templates
 415
 416 - builder:
 417     name: integration-start-cluster-run-test
 418     builders:
 419       - shell: !include-raw: integration-start-cluster-run-test.sh
 420
 421 # Macro: integration-install-robotframework
 422 # Operation: Installs robotframework using pip to a virtualenv
 423 # Used by: {project}-csit-* job templates
 424
 425 - builder:
 426     name: integration-install-robotframework
 427     builders:
 428       - shell: !include-raw: integration-install-robotframework.sh
 429
 430 # Macro: integration-cleanup-tmp
 431 # Operation: Cleans up temporary files created by build
 432 # Used by: {project}-csit-* job templates
 433
 434 - builder:
 435     name: integration-cleanup-tmp
 436     builders:
 437       - shell: !include-raw: integration-cleanup-tmp.sh
 438
 439 # Macro: integration-multipatch-builder
 440 # Operation: checks out multiple patches and builds custom distribution
 441 # Used by: integration-multipatch-test-* job templates
 442
 443 - builder:
 444     name: integration-multipatch-builder
 445     builders:
 446       - shell: !include-raw: multipatch-distribution.sh
 447
 448 # Macro: integration-cleanup-workspace
 449 # Operation: Cleans up files possibly left there by the previous build
 450 # Used by: {project}-csit-* job templates
 451
 452 - builder:
 453     name: integration-cleanup-workspace
 454     builders:
 455       - shell: !include-raw: integration-cleanup-workspace.sh
 456
 457 # Macro: integration-rebase-patch
 458 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
 459 # Used by: integration-patch-distribution-* job template
 460
 461 - builder:
 462     name: integration-rebase-gerrit-patch
 463     builders:
 464       - shell: !include-raw: integration-rebase-gerrit-patch.sh
 465
 466 # Macro: integration-install-common-functions
 467 # Operation: Copy the common-functions.sh script to csit nodes
 468 # Used by: {project}-csit-* job templates
 469 - builder:
 470     name: integration-install-common-functions
 471     builders:
 472       - shell: !include-raw:
 473           - copy-common-functions.sh
 474           - common-functions.sh
 475
 476 - builder:
 477     name: integration-deploy-openstack-run-test
 478     builders:
 479       - shell: !include-raw:
 480           - integration-deploy-openstack-run-test.sh
 481
 482 - builder:
 483     name: integration-compare-distributions
 484     # Operation: will compare an already created distribtion (probably last
 485     # published to nexus) to the current distribution being created.
 486     # The comparison is done to find the list of patches that are new to the
 487     # current distribution.
 488     # Required bash variables:
 489     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
 490     builders:
 491       - shell: !include-raw: integration-compare-distributions.sh
 492
 493 - builder:
 494     name: integration-upload-distribution
 495     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
 496     # Arguments:
 497     #   {dist-pom} path to pom file containing the version
 498     # Required bash variables:
 499     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 500     #     Use distribution-karaf-version macro to customize.
 501     builders:
 502       - config-file-provider:
 503           files:
 504             - file-id: global-settings
 505               variable: GLOBAL_SETTINGS_FILE
 506             - file-id: integration-settings
 507               variable: SETTINGS_FILE
 508       - integration-set-variables
 509       - inject:
 510           properties-content: BUNDLE_POM={dist-pom}
 511       - shell: !include-raw-escape: integration-upload-distribution.sh
 512       - inject:
 513           properties-file: integration-upload-distribution.env
 514
 515 ##############
 516 # Publishers #
 517 ##############
 518
 519 # Macro: integration-robot
 520 # Operation: this macro publishes robot results
 521 # Used by: {project}-csit-* job templates
 522
 523 - publisher:
 524     name: integration-robot
 525     publishers:
 526       - robot:
 527           output-path: ''
 528           other-files: ''
 529           unstable-threshold: '{unstable-if}'
 530           pass-threshold: '{pass-if}'
 531           only-critical: false
 532
 533 - publisher:
 534     name: integration-robot-tempest
 535     publishers:
 536       - robot:
 537           output-path: ''
 538           other-files:
 539             - tempest_results.html
 540           unstable-threshold: '{unstable-if}'
 541           pass-threshold: '{pass-if}'
 542           only-critical: false
 543
 544 - publisher:
 545     name: integration-csit-archive-build
 546     publishers:
 547       - postbuildscript:
 548           builders:
 549             - role: BOTH
 550               build-on:
 551                 - ABORTED
 552                 - FAILURE
 553                 - NOT_BUILT
 554                 - SUCCESS
 555                 - UNSTABLE
 556               build-steps:
 557                 - shell: |
 558                     #!/bin/bash
 559                     cd $WORKSPACE
 560                     mkdir -p ./archives
 561                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
 562                     unzip -d ./archives robot-plugin.zip
 563                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
 564                     # TODO: Can the following line ever fail?
 565                     find . -type f -name *.hprof -print0 \
 566                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
 567                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
 568           mark-unstable-if-failed: true
 569
 570
 571 - publisher:
 572     name: integration-csit-run-postscript
 573     publishers:
 574       - postbuildscript:
 575           builders:
 576             - role: BOTH
 577               build-on:
 578                 - ABORTED
 579                 - FAILURE
 580                 - NOT_BUILT
 581                 - SUCCESS
 582                 - UNSTABLE
 583               build-steps:
 584                 - config-file-provider:
 585                     files:
 586                       - file-id: 'odl-elastic-cloud'
 587                         target: '${HOME}/.netrc'
 588                 - shell: |
 589                     #!/bin/bash
 590                     #The following script runs any configurable postplan stored in test/csit/postplans.
 591                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
 592                         echo "postplan exists!!!"
 593                         echo "Changing the postplan path..."
 594                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
 595                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
 596                         cat postplan.txt
 597                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
 598                             echo "Executing ${line}..."
 599                             ${line}
 600                         done
 601                     fi
 602                     rm ${HOME}/.netrc
 603           mark-unstable-if-failed: true
 604
 605 - publisher:
 606     name: integration-csit-collect-netvirt-logs
 607     publishers:
 608       - postbuildscript:
 609           builders:
 610             - role: BOTH
 611               build-on:
 612                 - ABORTED
 613                 - FAILURE
 614                 - NOT_BUILT
 615                 - SUCCESS
 616                 - UNSTABLE
 617               build-steps:
 618                 - inject:
 619                     properties-file: 'slave_addresses.txt'
 620                 - shell: |
 621                     #!/bin/bash
 622                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
 623                     collect_netvirt_logs
 624           mark-unstable-if-failed: true