jjb/integration/integration-macros.yaml

   1 ---
   2 # TODO: Make all bash constants more readable.
   3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
   4
   5 - parameter:
   6     name: integration-csit-parameters
   7     parameters:
   8       - opendaylight-infra-parameters:
   9           os-cloud: '{os-cloud}'
  10           project: 'integration/test'
  11           branch: '{branch}'
  12           refspec: 'refs/heads/{branch}'
  13           artifacts: '{archive-artifacts}'
  14       # CSIT
  15       - integration-bundle-url:
  16           bundle-url: '{bundle-url}'
  17       - integration-repo-url:
  18           repo-url: '{repo-url}'
  19       - integration-test-options:
  20           test-options: '{robot-options}'
  21       - integration-controller-parameters:
  22           controller-debug-map: '{debug-map}'
  23           controller-features: '{install-features}'
  24           controller-scope: '{controller-scope}'
  25           controller-use-features-boot: '{use-features-boot}'
  26           feature-elasticsearch-attribute: '{elasticsearch-attribute}'
  27
  28 - parameter:
  29     name: integration-controller-parameters
  30     parameters:
  31       - string:
  32           name: CONTROLLERSCOPE
  33           default: '{controller-scope}'
  34           description: 'Feature Only or Feature All test'
  35
  36       - string:
  37           name: CONTROLLERFEATURES
  38           default: '{controller-features}'
  39           description: |
  40               Features to install in the controller separated by comma
  41               (note: extra features such as decanter are inserted in install scripts)
  42
  43       - string:
  44           name: CONTROLLERDEBUGMAP
  45           default: '{controller-debug-map}'
  46           description: |
  47               Key:Value map of debugs to enable on the controller separated by spaces.
  48               The module should not include org.opendaylight.
  49               The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
  50               (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
  51
  52       - string:
  53           name: ELASTICSEARCHATTRIBUTE
  54           default: '{feature-elasticsearch-attribute}'
  55           description: |
  56               Configure the period the script should harvest JVM metrics.
  57               Default is short.
  58               Possible values: long (its value is 120000)
  59                                short (its value is 5000)
  60                                any number between 5000 and 120000
  61
  62       - string:
  63           name: USEFEATURESBOOT
  64           default: '{controller-use-features-boot}'
  65           description: 'Boolean to toggle whether featuresBoot is appended or ignored'
  66
  67
  68 - parameter:
  69     name: integration-distribution-branch
  70     parameters:
  71       - string:
  72           name: DISTROBRANCH
  73           default: '{branch}'
  74           description: 'Distribution GIT branch'
  75
  76 - parameter:
  77     name: distribution-karaf-version
  78     # Arguments:
  79     #   karaf-version: Provide, otherwise defaults.yaml value is used.
  80     parameters:
  81       - string:
  82           name: KARAF_VERSION
  83           default: '{karaf-version}'
  84           description: 'karaf3, karaf4, odl, affects ODL deploy scripts'
  85
  86 - parameter:
  87     name: integration-bundle-url
  88     parameters:
  89       - string:
  90           name: BUNDLE_URL
  91           default: '{bundle-url}'
  92           description: 'URL to karaf distribution zip'
  93
  94 - parameter:
  95     name: integration-repo-url
  96     parameters:
  97       - string:
  98           name: REPO_URL
  99           default: '{repo-url}'
 100           description: 'URL to feature repo artifact'
 101
 102 - parameter:
 103     name: integration-jdk-version
 104     parameters:
 105       - string:
 106           name: JDKVERSION
 107           default: '{jdkversion}'
 108           description: 'Parameter to indicate JAVA Version'
 109
 110 - parameter:
 111     name: integration-distribution-stream
 112     parameters:
 113       - string:
 114           name: DISTROSTREAM
 115           default: '{stream}'
 116           description: 'Distribution stream string, for suites to know which behavior to expect'
 117
 118 - parameter:
 119     name: integration-script-plan
 120     parameters:
 121       - string:
 122           name: SCRIPTPLAN
 123           default: '{script-plan}'
 124           description: 'Script plan we will run before downloading and un-archiving am ODL distribution'
 125
 126 - parameter:
 127     name: integration-config-plan
 128     parameters:
 129       - string:
 130           name: CONFIGPLAN
 131           default: '{config-plan}'
 132           description: 'Config plan we will run after un-archiving and starting an ODL distribution'
 133
 134 - parameter:
 135     name: integration-stream-test-plan
 136     parameters:
 137       - string:
 138           name: STREAMTESTPLAN
 139           default: '{stream-test-plan}'
 140           description: 'Stream-specific test plan we will run'
 141
 142 - parameter:
 143     name: integration-test-plan
 144     parameters:
 145       - string:
 146           name: TESTPLAN
 147           default: '{test-plan}'
 148           description: 'General test plan we will run unless stream-specific one is found'
 149
 150 - parameter:
 151     name: integration-test-suites
 152     parameters:
 153       - string:
 154           name: SUITES
 155           default: '{test-suites}'
 156           description: |
 157               List of space-separated suites. Useful when running specific suites within a testplan.
 158               Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
 159               (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
 160                openstack/tempest/tempest.robot netvirt/elan/elan.robot).
 161
 162 - parameter:
 163     name: integration-test-options
 164     parameters:
 165       - string:
 166           name: TESTOPTIONS
 167           default: '{test-options}'
 168           description: 'Robot command options'
 169
 170 - parameter:
 171     name: integration-distribution-git-url
 172     parameters:
 173       - string:
 174           name: DISTROGITURL
 175           default: '{git-url}/integration/distribution'
 176           description: 'Distribution GIT URL (do not modify)'
 177
 178 - parameter:
 179     name: integration-patch-refspec
 180     parameters:
 181       - string:
 182           name: PATCHREFSPEC
 183           default: '{branch}'
 184           description: 'Integration Patch Refspec'
 185
 186 # Macro: integration-gerrit-scm
 187 # Operation: this macro downloads a project gerrit
 188 # Used by: all csit jobs
 189
 190 - scm:
 191     name: integration-gerrit-scm
 192     scm:
 193       - git:
 194           credentials-id: 'opendaylight-jenkins-ssh'
 195           url: '$GIT_BASE'
 196           basedir: '{basedir}'
 197           refspec: '{refspec}'
 198           branches:
 199             - 'origin/{branch}'
 200           skip-tag: true
 201           choosing-strategy: 'gerrit'
 202
 203 # Macro: integration-distribution-scm
 204 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
 205 # Used by: integration-patch-test-{stream} job template
 206
 207 - scm:
 208     name: integration-distribution-scm
 209     scm:
 210       - git:
 211           credentials-id: 'opendaylight-jenkins-ssh'
 212           basedir: 'distribution'
 213           url: '$DISTROGITURL'
 214           refspec: ''
 215           branches:
 216             - 'origin/{branch}'
 217           skip-tag: true
 218
 219 # Macro: integration-trigger-patch-submitted
 220 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
 221 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
 222 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
 223 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
 224 # Arguments:
 225 #     gerrit-server-name: name of gerrit server to listen to
 226 #     project: pattern to match triggering projects
 227 #     branch: triggering branch name
 228 #     files: pattern to match triggering filepaths
 229 - trigger:
 230     name: integration-trigger-patch-submitted
 231     triggers:
 232       - gerrit:
 233           server-name: '{gerrit-server-name}'
 234           trigger-on:
 235             - patchset-created-event:
 236                 exclude-drafts: true
 237                 exclude-trivial-rebase: false
 238                 exclude-no-code-change: false
 239             - draft-published-event
 240             - comment-added-contains-event:
 241                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 242           override-votes: true
 243           gerrit-build-unstable-verified-value: +1
 244           gerrit-build-unstable-codereview-value: 0
 245           projects:
 246             - project-compare-type: ANT
 247               project-pattern: '{project}'
 248               branches:
 249                 - branch-compare-type: ANT
 250                   branch-pattern: '**/{branch}'
 251               file-paths:
 252                 - compare-type: ANT
 253                   pattern: '{files}'
 254
 255 # Macro: integration-csit-only
 256 # Operation: this macro sets a trigger on reverse job list
 257 # Used by: csit -only- job templates
 258
 259 - trigger:
 260     name: integration-csit-trigger
 261     triggers:
 262       - reverse:
 263           jobs: '{jobs}'
 264           result: 'success'
 265
 266 # Macro: integration-openstack-controller-mininet
 267 # Operation: this macro will spin the controller and mininet vms
 268 # Used by: {project}-csit-* job templates
 269
 270 - wrapper:
 271     name: integration-openstack-controller-mininet
 272     wrappers:
 273       - openstack:
 274           instances:
 275             - cloud-name: 'ODLRPC'
 276               template-name: '{controller-image}'
 277               count: '{controller-vms}'
 278             - cloud-name: 'ODLRPC'
 279               template-name: '{mininet-image}'
 280               count: '{mininet-vms}'
 281
 282 # Macro: integration-openstack-controller-devstack
 283 # Operation: this macro will spin the controller and devstack vms
 284 # Used by: {project}-csit-openstack job templates
 285 - wrapper:
 286     name: integration-openstack-controller-devstack
 287     wrappers:
 288       - openstack:
 289           instances:
 290             - cloud-name: 'ODLRPC'
 291               template-name: '{controller-image}'
 292               count: '{controller-vms}'
 293             - cloud-name: 'ODLRPC'
 294               template-name: '{openstack-image}'
 295               count: '{openstack-vms}'
 296
 297 #################
 298 # Shell Scripts #
 299 #################
 300
 301 # Macro: integration-get-apex-addresses
 302 # Operation: this macro gets the IP addresses of the dynamic vms from apex snapshots
 303 # Used by: {project}-csit-* apex related job templates
 304
 305 - builder:
 306     name: integration-get-apex-addresses
 307     builders:
 308       - shell: !include-raw: integration-get-apex-addresses.sh
 309
 310 # Macro: integration-apex-run-tests
 311 # Operation: this macro just runs the robot suites
 312 # Used by: {project}-csit-* apex related job templates
 313
 314 - builder:
 315     name: integration-apex-run-tests
 316     builders:
 317       - shell: !include-raw: integration-apex-run-tests.sh
 318
 319 # Macro: integration-get-slave-addresses
 320 # Operation: this macro gets the IP addresses of the dynamic vms
 321 # Used by: {project}-csit-* job templates
 322
 323 - builder:
 324     name: integration-get-slave-addresses
 325     builders:
 326       - shell: !include-raw: integration-get-slave-addresses.sh
 327
 328 - builder:
 329     name: integration-set-variables
 330     # Operation: Construct variables not requiring online detection and export using set_variables.env
 331     # Required bash variables:
 332     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 333     builders:
 334       - shell: !include-raw: integration-set-variables.sh
 335       - inject:
 336           properties-file: set_variables.env
 337
 338 - builder:
 339     name: integration-detect-variables
 340     # Operation: Construct variables including online detection and export using detect_variables.env
 341     #   This calls integration-set-variables internally.
 342     # Required bash variables:
 343     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 344     builders:
 345       - integration-set-variables
 346       - shell: !include-raw: integration-detect-variables.sh
 347       - inject:
 348           properties-file: detect_variables.env
 349
 350 # Macro: integration-deply-controller-run-test
 351 # Operation: this macro deploys single contoller and runs test
 352 # Used by: {project}-csit-1node-* job templates
 353
 354 - builder:
 355     name: integration-deploy-controller-run-test
 356     builders:
 357       - shell: !include-raw: integration-deploy-controller-run-test.sh
 358
 359 # Macro: integration-configure-clustering
 360 # Operation: this macro configures the clustering
 361 # Used by: {project}-csit-3node-* job templates
 362
 363 - builder:
 364     name: integration-configure-clustering
 365     builders:
 366       - shell: !include-raw: integration-configure-clustering.sh
 367
 368 # Macro: integration-start-cluster-run-test
 369 # Operation: this macro starts the 3-node cluster and runs test
 370 # Used by: {project}-csit-3node-* job templates
 371
 372 - builder:
 373     name: integration-start-cluster-run-test
 374     builders:
 375       - shell: !include-raw: integration-start-cluster-run-test.sh
 376
 377 # Macro: integration-install-robotframework
 378 # Operation: Installs robotframework using pip to a virtualenv
 379 # Used by: {project}-csit-* job templates
 380
 381 - builder:
 382     name: integration-install-robotframework
 383     builders:
 384       - shell: !include-raw: integration-install-robotframework.sh
 385
 386 # Macro: integration-cleanup-tmp
 387 # Operation: Cleans up temporary files created by build
 388 # Used by: {project}-csit-* job templates
 389
 390 - builder:
 391     name: integration-cleanup-tmp
 392     builders:
 393       - shell: !include-raw: integration-cleanup-tmp.sh
 394
 395 # Macro: integration-multipatch-builder
 396 # Operation: checks out multiple patches and builds custom distribution
 397 # Used by: integration-multipatch-test-* job templates
 398
 399 - builder:
 400     name: integration-multipatch-builder
 401     builders:
 402       - shell: !include-raw: multipatch-distribution.sh
 403
 404 # Macro: integration-cleanup-workspace
 405 # Operation: Cleans up files possibly left there by the previous build
 406 # Used by: {project}-csit-* job templates
 407
 408 - builder:
 409     name: integration-cleanup-workspace
 410     builders:
 411       - shell: !include-raw: integration-cleanup-workspace.sh
 412
 413 # Macro: integration-rebase-patch
 414 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
 415 # Used by: integration-patch-distribution-* job template
 416
 417 - builder:
 418     name: integration-rebase-gerrit-patch
 419     builders:
 420       - shell: !include-raw: integration-rebase-gerrit-patch.sh
 421
 422 # Macro: integration-install-common-functions
 423 # Operation: Copy the common-functions.sh script to csit nodes
 424 # Used by: {project}-csit-* job templates
 425 - builder:
 426     name: integration-install-common-functions
 427     builders:
 428       - shell: !include-raw:
 429           - copy-common-functions.sh
 430           - common-functions.sh
 431
 432 - builder:
 433     name: integration-deploy-openstack-run-test
 434     builders:
 435       - shell: !include-raw:
 436           - integration-deploy-openstack-run-test.sh
 437
 438 - builder:
 439     name: integration-apex-image-manager
 440     builders:
 441       - shell: !include-raw:
 442           - ../opendaylight-infra-apex-image-management.sh
 443
 444 - builder:
 445     name: integration-compare-distributions
 446     # Operation: will compare an already created distribtion (probably last
 447     # published to nexus) to the current distribution being created.
 448     # The comparison is done to find the list of patches that are new to the
 449     # current distribution.
 450     # Required bash variables:
 451     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
 452     builders:
 453       - shell: !include-raw: integration-compare-distributions.sh
 454
 455 - builder:
 456     name: integration-upload-distribution
 457     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
 458     # Arguments:
 459     #   {dist-pom} path to pom file containing the version
 460     # Required bash variables:
 461     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 462     #     Use distribution-karaf-version macro to customize.
 463     builders:
 464       - config-file-provider:
 465           files:
 466             - file-id: global-settings
 467               variable: GLOBAL_SETTINGS_FILE
 468             - file-id: integration-settings
 469               variable: SETTINGS_FILE
 470       - integration-set-variables
 471       - inject:
 472           properties-content: BUNDLE_POM={dist-pom}
 473       - shell: !include-raw-escape: integration-upload-distribution.sh
 474       - inject:
 475           properties-file: integration-upload-distribution.env
 476
 477 ##############
 478 # Publishers #
 479 ##############
 480
 481 - publisher:
 482     name: integration-csit-publish
 483     publishers:
 484       - robot:
 485           output-path: ''
 486           other-files: '{obj:robot_other_files}'
 487           unstable-threshold: '{robot-unstable-threshold}'
 488           pass-threshold: '{robot-pass-threshold}'
 489           only-critical: false
 490       - integration-csit-archive-build
 491       - integration-csit-run-postscript
 492       - releng-openstack-stack-delete
 493
 494 - publisher:
 495     name: integration-csit-archive-build
 496     publishers:
 497       - postbuildscript:
 498           builders:
 499             - role: BOTH
 500               build-on:
 501                 - ABORTED
 502                 - FAILURE
 503                 - NOT_BUILT
 504                 - SUCCESS
 505                 - UNSTABLE
 506               build-steps:
 507                 - shell: |
 508                     #!/bin/bash
 509                     echo "Archiving csit artifacts"
 510                     cd $WORKSPACE
 511                     mkdir -p ./archives
 512                     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
 513                         NODE_FOLDER="./archives/odl_${i}"
 514                         mkdir -p ${NODE_FOLDER}
 515                         mv odl${i}_* ${NODE_FOLDER} || true
 516                         mv karaf_${i}_*_threads* ${NODE_FOLDER} || true
 517                         mv *_${i}.png ${NODE_FOLDER} || true
 518                         mv /tmp/odl${i}_* ${NODE_FOLDER} || true
 519                         mv gclogs-${i} ${NODE_FOLDER} || true
 520                     done
 521                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
 522                     unzip -d ./archives robot-plugin.zip
 523                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
 524                     # TODO: Can the following line ever fail?
 525                     find . -type f -name *.hprof -print0 \
 526                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
 527                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
 528           mark-unstable-if-failed: true
 529
 530
 531 - publisher:
 532     name: integration-csit-run-postscript
 533     publishers:
 534       - postbuildscript:
 535           builders:
 536             - role: BOTH
 537               build-on:
 538                 - ABORTED
 539                 - FAILURE
 540                 - NOT_BUILT
 541                 - SUCCESS
 542                 - UNSTABLE
 543               build-steps:
 544                 - config-file-provider:
 545                     files:
 546                       - file-id: 'odl-elastic-cloud'
 547                         target: '${HOME}/.netrc'
 548                 - shell: |
 549                     #!/bin/bash
 550                     #The following script runs any configurable postplan stored in test/csit/postplans.
 551                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
 552                         echo "postplan exists!!!"
 553                         echo "Changing the postplan path..."
 554                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
 555                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
 556                         cat postplan.txt
 557                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
 558                             echo "Executing ${line}..."
 559                             ${line}
 560                         done
 561                     fi
 562                     rm ${HOME}/.netrc
 563           mark-unstable-if-failed: true
 564
 565 - publisher:
 566     name: integration-csit-collect-netvirt-logs
 567     publishers:
 568       - postbuildscript:
 569           builders:
 570             - role: BOTH
 571               build-on:
 572                 - ABORTED
 573                 - FAILURE
 574                 - NOT_BUILT
 575                 - SUCCESS
 576                 - UNSTABLE
 577               build-steps:
 578                 - inject:
 579                     properties-file: 'slave_addresses.txt'
 580                 - shell: |
 581                     #!/bin/bash
 582                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
 583                     collect_netvirt_logs
 584           mark-unstable-if-failed: true