jjb/integration/integration-macros.yaml

   1 ---
   2 # TODO: Make all bash constants more readable.
   3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
   4
   5 - parameter:
   6     name: integration-csit-parameters
   7     parameters:
   8       - opendaylight-infra-parameters:
   9           os-cloud: '{os-cloud}'
  10           project: 'integration/test'
  11           branch: '{branch}'
  12           refspec: 'refs/heads/{branch}'
  13           artifacts: '{archive-artifacts}'
  14       # CSIT
  15       - integration-bundle-url:
  16           bundle-url: '{bundle-url}'
  17       - integration-repo-url:
  18           repo-url: '{repo-url}'
  19       - integration-test-options:
  20           test-options: '{robot-options}'
  21       - integration-controller-parameters:
  22           controller-debug-map: '{debug-map}'
  23           controller-features: '{install-features}'
  24           controller-scope: '{controller-scope}'
  25           controller-use-features-boot: '{use-features-boot}'
  26
  27 - parameter:
  28     name: integration-controller-parameters
  29     parameters:
  30       - string:
  31           name: CONTROLLERSCOPE
  32           default: '{controller-scope}'
  33           description: 'Feature Only or Feature All test'
  34
  35       - string:
  36           name: CONTROLLERFEATURES
  37           default: '{controller-features}'
  38           description: 'Features to install in the controller separated by comma'
  39
  40       - string:
  41           name: CONTROLLERDEBUGMAP
  42           default: '{controller-debug-map}'
  43           description: |
  44               Key:Value map of debugs to enable on the controller separated by spaces.
  45               The module should not include org.opendaylight.
  46               The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
  47               (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
  48
  49       - string:
  50           name: USEFEATURESBOOT
  51           default: '{controller-use-features-boot}'
  52           description: 'Boolean to toggle whether featuresBoot is appended or ignored'
  53
  54
  55 - parameter:
  56     name: integration-distribution-branch
  57     parameters:
  58       - string:
  59           name: DISTROBRANCH
  60           default: '{branch}'
  61           description: 'Distribution GIT branch'
  62
  63 - parameter:
  64     name: distribution-karaf-version
  65     # Arguments:
  66     #   karaf-version: Provide, otherwise defaults.yaml value is used.
  67     parameters:
  68       - string:
  69           name: KARAF_VERSION
  70           default: '{karaf-version}'
  71           description: 'karaf3, karaf4, odl, affects ODL deploy scripts'
  72
  73 - parameter:
  74     name: integration-bundle-url
  75     parameters:
  76       - string:
  77           name: BUNDLE_URL
  78           default: '{bundle-url}'
  79           description: 'URL to karaf distribution zip'
  80
  81 - parameter:
  82     name: integration-repo-url
  83     parameters:
  84       - string:
  85           name: REPO_URL
  86           default: '{repo-url}'
  87           description: 'URL to feature repo artifact'
  88
  89 - parameter:
  90     name: integration-jdk-version
  91     parameters:
  92       - string:
  93           name: JDKVERSION
  94           default: '{jdkversion}'
  95           description: 'Parameter to indicate JAVA Version'
  96
  97 - parameter:
  98     name: integration-distribution-stream
  99     parameters:
 100       - string:
 101           name: DISTROSTREAM
 102           default: '{stream}'
 103           description: 'Distribution stream string, for suites to know which behavior to expect'
 104
 105 - parameter:
 106     name: integration-script-plan
 107     parameters:
 108       - string:
 109           name: SCRIPTPLAN
 110           default: '{script-plan}'
 111           description: 'Script plan we will run before downloading and un-archiving am ODL distribution'
 112
 113 - parameter:
 114     name: integration-config-plan
 115     parameters:
 116       - string:
 117           name: CONFIGPLAN
 118           default: '{config-plan}'
 119           description: 'Config plan we will run after un-archiving and starting an ODL distribution'
 120
 121 - parameter:
 122     name: integration-stream-test-plan
 123     parameters:
 124       - string:
 125           name: STREAMTESTPLAN
 126           default: '{stream-test-plan}'
 127           description: 'Stream-specific test plan we will run'
 128
 129 - parameter:
 130     name: integration-test-plan
 131     parameters:
 132       - string:
 133           name: TESTPLAN
 134           default: '{test-plan}'
 135           description: 'General test plan we will run unless stream-specific one is found'
 136
 137 - parameter:
 138     name: integration-test-suites
 139     parameters:
 140       - string:
 141           name: SUITES
 142           default: '{test-suites}'
 143           description: |
 144               List of space-separated suites. Useful when running specific suites within a testplan.
 145               Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
 146               (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
 147                openstack/tempest/tempest.robot netvirt/elan/elan.robot).
 148
 149 - parameter:
 150     name: integration-test-options
 151     parameters:
 152       - string:
 153           name: TESTOPTIONS
 154           default: '{test-options}'
 155           description: 'Robot command options'
 156
 157 - parameter:
 158     name: integration-distribution-git-url
 159     parameters:
 160       - string:
 161           name: DISTROGITURL
 162           default: '{git-url}/integration/distribution'
 163           description: 'Distribution GIT URL (do not modify)'
 164
 165 - parameter:
 166     name: integration-patch-refspec
 167     parameters:
 168       - string:
 169           name: PATCHREFSPEC
 170           default: '{branch}'
 171           description: 'Integration Patch Refspec'
 172
 173 # Macro: integration-gerrit-scm
 174 # Operation: this macro downloads a project gerrit
 175 # Used by: all csit jobs
 176
 177 - scm:
 178     name: integration-gerrit-scm
 179     scm:
 180       - git:
 181           credentials-id: 'opendaylight-jenkins-ssh'
 182           url: '$GIT_BASE'
 183           basedir: '{basedir}'
 184           refspec: '{refspec}'
 185           branches:
 186             - 'origin/{branch}'
 187           skip-tag: true
 188           choosing-strategy: 'gerrit'
 189
 190 # Macro: integration-distribution-scm
 191 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
 192 # Used by: integration-patch-test-{stream} job template
 193
 194 - scm:
 195     name: integration-distribution-scm
 196     scm:
 197       - git:
 198           credentials-id: 'opendaylight-jenkins-ssh'
 199           basedir: 'distribution'
 200           url: '$DISTROGITURL'
 201           refspec: ''
 202           branches:
 203             - 'origin/{branch}'
 204           skip-tag: true
 205
 206 # Macro: integration-trigger-patch-submitted
 207 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
 208 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
 209 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
 210 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
 211 # Arguments:
 212 #     gerrit-server-name: name of gerrit server to listen to
 213 #     project: pattern to match triggering projects
 214 #     branch: triggering branch name
 215 #     files: pattern to match triggering filepaths
 216 - trigger:
 217     name: integration-trigger-patch-submitted
 218     triggers:
 219       - gerrit:
 220           server-name: '{gerrit-server-name}'
 221           trigger-on:
 222             - patchset-created-event:
 223                 exclude-drafts: true
 224                 exclude-trivial-rebase: false
 225                 exclude-no-code-change: false
 226             - draft-published-event
 227             - comment-added-contains-event:
 228                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 229           override-votes: true
 230           gerrit-build-unstable-verified-value: +1
 231           gerrit-build-unstable-codereview-value: 0
 232           projects:
 233             - project-compare-type: ANT
 234               project-pattern: '{project}'
 235               branches:
 236                 - branch-compare-type: ANT
 237                   branch-pattern: '**/{branch}'
 238               file-paths:
 239                 - compare-type: ANT
 240                   pattern: '{files}'
 241
 242 # Macro: integration-csit-only
 243 # Operation: this macro sets a trigger on reverse job list
 244 # Used by: csit -only- job templates
 245
 246 - trigger:
 247     name: integration-csit-trigger
 248     triggers:
 249       - reverse:
 250           jobs: '{jobs}'
 251           result: 'success'
 252
 253 # Macro: integration-openstack-controller-mininet
 254 # Operation: this macro will spin the controller and mininet vms
 255 # Used by: {project}-csit-* job templates
 256
 257 - wrapper:
 258     name: integration-openstack-controller-mininet
 259     wrappers:
 260       - openstack:
 261           instances:
 262             - cloud-name: 'ODLRPC'
 263               template-name: '{controller-image}'
 264               count: '{controller-vms}'
 265             - cloud-name: 'ODLRPC'
 266               template-name: '{mininet-image}'
 267               count: '{mininet-vms}'
 268
 269 # Macro: integration-openstack-controller-devstack
 270 # Operation: this macro will spin the controller and devstack vms
 271 # Used by: {project}-csit-openstack job templates
 272 - wrapper:
 273     name: integration-openstack-controller-devstack
 274     wrappers:
 275       - openstack:
 276           instances:
 277             - cloud-name: 'ODLRPC'
 278               template-name: '{controller-image}'
 279               count: '{controller-vms}'
 280             - cloud-name: 'ODLRPC'
 281               template-name: '{openstack-image}'
 282               count: '{openstack-vms}'
 283
 284 #################
 285 # Shell Scripts #
 286 #################
 287
 288 # Macro: integration-get-apex-addresses
 289 # Operation: this macro gets the IP addresses of the dynamic vms from apex snapshots
 290 # Used by: {project}-csit-* apex related job templates
 291
 292 - builder:
 293     name: integration-get-apex-addresses
 294     builders:
 295       - shell: !include-raw: integration-get-apex-addresses.sh
 296
 297 # Macro: integration-apex-run-tests
 298 # Operation: this macro just runs the robot suites
 299 # Used by: {project}-csit-* apex related job templates
 300
 301 - builder:
 302     name: integration-apex-run-tests
 303     builders:
 304       - shell: !include-raw: integration-apex-run-tests.sh
 305
 306 # Macro: integration-get-slave-addresses
 307 # Operation: this macro gets the IP addresses of the dynamic vms
 308 # Used by: {project}-csit-* job templates
 309
 310 - builder:
 311     name: integration-get-slave-addresses
 312     builders:
 313       - shell: !include-raw: integration-get-slave-addresses.sh
 314
 315 - builder:
 316     name: integration-set-variables
 317     # Operation: Construct variables not requiring online detection and export using set_variables.env
 318     # Required bash variables:
 319     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 320     builders:
 321       - shell: !include-raw: integration-set-variables.sh
 322       - inject:
 323           properties-file: set_variables.env
 324
 325 - builder:
 326     name: integration-detect-variables
 327     # Operation: Construct variables including online detection and export using detect_variables.env
 328     #   This calls integration-set-variables internally.
 329     # Required bash variables:
 330     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 331     builders:
 332       - integration-set-variables
 333       - shell: !include-raw: integration-detect-variables.sh
 334       - inject:
 335           properties-file: detect_variables.env
 336
 337 # Macro: integration-deply-controller-run-test
 338 # Operation: this macro deploys single contoller and runs test
 339 # Used by: {project}-csit-1node-* job templates
 340
 341 - builder:
 342     name: integration-deploy-controller-run-test
 343     builders:
 344       - shell: !include-raw: integration-deploy-controller-run-test.sh
 345
 346 # Macro: integration-configure-clustering
 347 # Operation: this macro configures the clustering
 348 # Used by: {project}-csit-3node-* job templates
 349
 350 - builder:
 351     name: integration-configure-clustering
 352     builders:
 353       - shell: !include-raw: integration-configure-clustering.sh
 354
 355 # Macro: integration-start-cluster-run-test
 356 # Operation: this macro starts the 3-node cluster and runs test
 357 # Used by: {project}-csit-3node-* job templates
 358
 359 - builder:
 360     name: integration-start-cluster-run-test
 361     builders:
 362       - shell: !include-raw: integration-start-cluster-run-test.sh
 363
 364 # Macro: integration-install-robotframework
 365 # Operation: Installs robotframework using pip to a virtualenv
 366 # Used by: {project}-csit-* job templates
 367
 368 - builder:
 369     name: integration-install-robotframework
 370     builders:
 371       - shell: !include-raw: integration-install-robotframework.sh
 372
 373 # Macro: integration-cleanup-tmp
 374 # Operation: Cleans up temporary files created by build
 375 # Used by: {project}-csit-* job templates
 376
 377 - builder:
 378     name: integration-cleanup-tmp
 379     builders:
 380       - shell: !include-raw: integration-cleanup-tmp.sh
 381
 382 # Macro: integration-multipatch-builder
 383 # Operation: checks out multiple patches and builds custom distribution
 384 # Used by: integration-multipatch-test-* job templates
 385
 386 - builder:
 387     name: integration-multipatch-builder
 388     builders:
 389       - shell: !include-raw: multipatch-distribution.sh
 390
 391 # Macro: integration-cleanup-workspace
 392 # Operation: Cleans up files possibly left there by the previous build
 393 # Used by: {project}-csit-* job templates
 394
 395 - builder:
 396     name: integration-cleanup-workspace
 397     builders:
 398       - shell: !include-raw: integration-cleanup-workspace.sh
 399
 400 # Macro: integration-rebase-patch
 401 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
 402 # Used by: integration-patch-distribution-* job template
 403
 404 - builder:
 405     name: integration-rebase-gerrit-patch
 406     builders:
 407       - shell: !include-raw: integration-rebase-gerrit-patch.sh
 408
 409 # Macro: integration-install-common-functions
 410 # Operation: Copy the common-functions.sh script to csit nodes
 411 # Used by: {project}-csit-* job templates
 412 - builder:
 413     name: integration-install-common-functions
 414     builders:
 415       - shell: !include-raw:
 416           - copy-common-functions.sh
 417           - common-functions.sh
 418
 419 - builder:
 420     name: integration-deploy-openstack-run-test
 421     builders:
 422       - shell: !include-raw:
 423           - integration-deploy-openstack-run-test.sh
 424
 425 - builder:
 426     name: integration-apex-image-manager
 427     builders:
 428       - shell: !include-raw:
 429           - ../opendaylight-infra-apex-image-management.sh
 430
 431 - builder:
 432     name: integration-compare-distributions
 433     # Operation: will compare an already created distribtion (probably last
 434     # published to nexus) to the current distribution being created.
 435     # The comparison is done to find the list of patches that are new to the
 436     # current distribution.
 437     # Required bash variables:
 438     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
 439     builders:
 440       - shell: !include-raw: integration-compare-distributions.sh
 441
 442 - builder:
 443     name: integration-upload-distribution
 444     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
 445     # Arguments:
 446     #   {dist-pom} path to pom file containing the version
 447     # Required bash variables:
 448     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 449     #     Use distribution-karaf-version macro to customize.
 450     builders:
 451       - config-file-provider:
 452           files:
 453             - file-id: global-settings
 454               variable: GLOBAL_SETTINGS_FILE
 455             - file-id: integration-settings
 456               variable: SETTINGS_FILE
 457       - integration-set-variables
 458       - inject:
 459           properties-content: BUNDLE_POM={dist-pom}
 460       - shell: !include-raw-escape: integration-upload-distribution.sh
 461       - inject:
 462           properties-file: integration-upload-distribution.env
 463
 464 ##############
 465 # Publishers #
 466 ##############
 467
 468 - publisher:
 469     name: integration-csit-publish
 470     publishers:
 471       - robot:
 472           output-path: ''
 473           other-files: '{obj:robot_other_files}'
 474           unstable-threshold: '{robot-unstable-threshold}'
 475           pass-threshold: '{robot-pass-threshold}'
 476           only-critical: false
 477       - integration-csit-archive-build
 478       - integration-csit-run-postscript
 479       - releng-openstack-stack-delete
 480
 481 - publisher:
 482     name: integration-csit-archive-build
 483     publishers:
 484       - postbuildscript:
 485           builders:
 486             - role: BOTH
 487               build-on:
 488                 - ABORTED
 489                 - FAILURE
 490                 - NOT_BUILT
 491                 - SUCCESS
 492                 - UNSTABLE
 493               build-steps:
 494                 - shell: |
 495                     #!/bin/bash
 496                     echo "Archiving csit artifacts"
 497                     cd $WORKSPACE
 498                     mkdir -p ./archives
 499                     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
 500                         NODE_FOLDER="./archives/odl_${i}"
 501                         mkdir -p ${NODE_FOLDER}
 502                         mv odl${i}_* ${NODE_FOLDER} || true
 503                         mv karaf_${i}_*_threads* ${NODE_FOLDER} || true
 504                         mv *_${i}.png ${NODE_FOLDER} || true
 505                         mv /tmp/odl${i}_* ${NODE_FOLDER} || true
 506                     done
 507                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
 508                     unzip -d ./archives robot-plugin.zip
 509                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
 510                     # TODO: Can the following line ever fail?
 511                     find . -type f -name *.hprof -print0 \
 512                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
 513                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
 514           mark-unstable-if-failed: true
 515
 516
 517 - publisher:
 518     name: integration-csit-run-postscript
 519     publishers:
 520       - postbuildscript:
 521           builders:
 522             - role: BOTH
 523               build-on:
 524                 - ABORTED
 525                 - FAILURE
 526                 - NOT_BUILT
 527                 - SUCCESS
 528                 - UNSTABLE
 529               build-steps:
 530                 - config-file-provider:
 531                     files:
 532                       - file-id: 'odl-elastic-cloud'
 533                         target: '${HOME}/.netrc'
 534                 - shell: |
 535                     #!/bin/bash
 536                     #The following script runs any configurable postplan stored in test/csit/postplans.
 537                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
 538                         echo "postplan exists!!!"
 539                         echo "Changing the postplan path..."
 540                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
 541                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
 542                         cat postplan.txt
 543                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
 544                             echo "Executing ${line}..."
 545                             ${line}
 546                         done
 547                     fi
 548                     rm ${HOME}/.netrc
 549           mark-unstable-if-failed: true
 550
 551 - publisher:
 552     name: integration-csit-collect-netvirt-logs
 553     publishers:
 554       - postbuildscript:
 555           builders:
 556             - role: BOTH
 557               build-on:
 558                 - ABORTED
 559                 - FAILURE
 560                 - NOT_BUILT
 561                 - SUCCESS
 562                 - UNSTABLE
 563               build-steps:
 564                 - inject:
 565                     properties-file: 'slave_addresses.txt'
 566                 - shell: |
 567                     #!/bin/bash
 568                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
 569                     collect_netvirt_logs
 570           mark-unstable-if-failed: true