jjb/integration/integration-macros.yaml

   1 ---
   2 # TODO: Make all bash constants more readable.
   3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
   4
   5 - parameter:
   6     name: integration-csit-parameters
   7     parameters:
   8       - opendaylight-infra-parameters:
   9           os-cloud: '{os-cloud}'
  10           project: 'integration/test'
  11           branch: '{branch}'
  12           refspec: 'refs/heads/{branch}'
  13           artifacts: '{archive-artifacts}'
  14       # CSIT
  15       - integration-bundle-url:
  16           bundle-url: '{bundle-url}'
  17       - integration-repo-url:
  18           repo-url: '{repo-url}'
  19       - integration-test-options:
  20           test-options: '{robot-options}'
  21       - integration-controller-parameters:
  22           controller-debug-map: '{debug-map}'
  23           controller-features: '{install-features}'
  24           controller-max-mem: '{controller-max-mem}'
  25           controller-scope: '{controller-scope}'
  26           controller-use-features-boot: '{use-features-boot}'
  27           feature-elasticsearch-attribute: '{elasticsearch-attribute}'
  28
  29 - parameter:
  30     name: integration-controller-parameters
  31     parameters:
  32       - string:
  33           name: CONTROLLERSCOPE
  34           default: '{controller-scope}'
  35           description: 'Feature Only or Feature All test'
  36
  37       - string:
  38           name: CONTROLLERFEATURES
  39           default: '{controller-features}'
  40           description: |
  41               Features to install in the controller separated by comma.
  42               (note: Do not include decanter* features used for harvesting JVM metrics.
  43                They are inserted in install scripts.)
  44
  45       - string:
  46           name: CONTROLLERDEBUGMAP
  47           default: '{controller-debug-map}'
  48           description: |
  49               Key:Value map of debugs to enable on the controller separated by spaces.
  50               The module should not include org.opendaylight.
  51               The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
  52               (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
  53
  54       - string:
  55           name: CONTROLLERMAXMEM
  56           default: '{controller-max-mem}'
  57           description: |
  58               Max Heap size to use when starting the controller.
  59
  60       - string:
  61           name: ELASTICSEARCHATTRIBUTE
  62           default: '{feature-elasticsearch-attribute}'
  63           description: |
  64               Configure the period the script should harvest JVM metrics.
  65               Possible values: long (=120000)
  66                                short (=5000)
  67                                any number between 5000 and 120000
  68                                disabled (use for disabling harvesting)
  69
  70       - string:
  71           name: USEFEATURESBOOT
  72           default: '{controller-use-features-boot}'
  73           description: 'Boolean to toggle whether featuresBoot is appended or ignored'
  74
  75
  76 - parameter:
  77     name: integration-distribution-branch
  78     parameters:
  79       - string:
  80           name: DISTROBRANCH
  81           default: '{branch}'
  82           description: 'Distribution GIT branch'
  83
  84 - parameter:
  85     name: distribution-karaf-version
  86     # Arguments:
  87     #   karaf-version: Provide, otherwise defaults.yaml value is used.
  88     parameters:
  89       - string:
  90           name: KARAF_VERSION
  91           default: '{karaf-version}'
  92           description: 'karaf3, karaf4, odl, affects ODL deploy scripts'
  93
  94 - parameter:
  95     name: integration-bundle-url
  96     parameters:
  97       - string:
  98           name: BUNDLE_URL
  99           default: '{bundle-url}'
 100           description: 'URL to karaf distribution zip'
 101
 102 - parameter:
 103     name: integration-repo-url
 104     parameters:
 105       - string:
 106           name: REPO_URL
 107           default: '{repo-url}'
 108           description: 'URL to feature repo artifact'
 109
 110 - parameter:
 111     name: integration-jdk-version
 112     parameters:
 113       - string:
 114           name: JDKVERSION
 115           default: '{jdkversion}'
 116           description: 'Parameter to indicate JAVA Version'
 117
 118 - parameter:
 119     name: integration-distribution-stream
 120     parameters:
 121       - string:
 122           name: DISTROSTREAM
 123           default: '{stream}'
 124           description: 'Distribution stream string, for suites to know which behavior to expect'
 125
 126 - parameter:
 127     name: integration-script-plan
 128     parameters:
 129       - string:
 130           name: SCRIPTPLAN
 131           default: '{script-plan}'
 132           description: 'Script plan we will run before downloading and un-archiving am ODL distribution'
 133
 134 - parameter:
 135     name: integration-config-plan
 136     parameters:
 137       - string:
 138           name: CONFIGPLAN
 139           default: '{config-plan}'
 140           description: 'Config plan we will run after un-archiving and starting an ODL distribution'
 141
 142 - parameter:
 143     name: integration-stream-test-plan
 144     parameters:
 145       - string:
 146           name: STREAMTESTPLAN
 147           default: '{stream-test-plan}'
 148           description: 'Stream-specific test plan we will run'
 149
 150 - parameter:
 151     name: integration-test-plan
 152     parameters:
 153       - string:
 154           name: TESTPLAN
 155           default: '{test-plan}'
 156           description: 'General test plan we will run unless stream-specific one is found'
 157
 158 - parameter:
 159     name: integration-test-suites
 160     parameters:
 161       - string:
 162           name: SUITES
 163           default: '{test-suites}'
 164           description: |
 165               List of space-separated suites. Useful when running specific suites within a testplan.
 166               Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
 167               (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
 168                openstack/tempest/tempest.robot netvirt/elan/elan.robot).
 169
 170 - parameter:
 171     name: integration-test-options
 172     parameters:
 173       - string:
 174           name: TESTOPTIONS
 175           default: '{test-options}'
 176           description: 'Robot command options'
 177
 178 - parameter:
 179     name: integration-distribution-git-url
 180     parameters:
 181       - string:
 182           name: DISTROGITURL
 183           default: '{git-url}/integration/distribution'
 184           description: 'Distribution GIT URL (do not modify)'
 185
 186 - parameter:
 187     name: integration-patch-refspec
 188     parameters:
 189       - string:
 190           name: PATCHREFSPEC
 191           default: '{branch}'
 192           description: 'Integration Patch Refspec'
 193
 194 # Macro: integration-gerrit-scm
 195 # Operation: this macro downloads a project gerrit
 196 # Used by: all csit jobs
 197
 198 - scm:
 199     name: integration-gerrit-scm
 200     scm:
 201       - git:
 202           credentials-id: 'opendaylight-jenkins-ssh'
 203           url: '$GIT_BASE'
 204           basedir: '{basedir}'
 205           refspec: '{refspec}'
 206           branches:
 207             - 'origin/{branch}'
 208           skip-tag: true
 209           choosing-strategy: 'gerrit'
 210
 211 # Macro: integration-distribution-scm
 212 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
 213 # Used by: integration-patch-test-{stream} job template
 214
 215 - scm:
 216     name: integration-distribution-scm
 217     scm:
 218       - git:
 219           credentials-id: 'opendaylight-jenkins-ssh'
 220           basedir: 'distribution'
 221           url: '$DISTROGITURL'
 222           refspec: ''
 223           branches:
 224             - 'origin/{branch}'
 225           skip-tag: true
 226
 227 # Macro: integration-trigger-patch-submitted
 228 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
 229 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
 230 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
 231 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
 232 # Arguments:
 233 #     gerrit-server-name: name of gerrit server to listen to
 234 #     project: pattern to match triggering projects
 235 #     branch: triggering branch name
 236 #     files: pattern to match triggering filepaths
 237 - trigger:
 238     name: integration-trigger-patch-submitted
 239     triggers:
 240       - gerrit:
 241           server-name: '{gerrit-server-name}'
 242           trigger-on:
 243             - patchset-created-event:
 244                 exclude-drafts: true
 245                 exclude-trivial-rebase: false
 246                 exclude-no-code-change: false
 247             - draft-published-event
 248             - comment-added-contains-event:
 249                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 250           override-votes: true
 251           gerrit-build-unstable-verified-value: +1
 252           gerrit-build-unstable-codereview-value: 0
 253           projects:
 254             - project-compare-type: ANT
 255               project-pattern: '{project}'
 256               branches:
 257                 - branch-compare-type: ANT
 258                   branch-pattern: '**/{branch}'
 259               file-paths:
 260                 - compare-type: ANT
 261                   pattern: '{files}'
 262
 263 # Macro: integration-csit-only
 264 # Operation: this macro sets a trigger on reverse job list
 265 # Used by: csit -only- job templates
 266
 267 - trigger:
 268     name: integration-csit-trigger
 269     triggers:
 270       - reverse:
 271           jobs: '{jobs}'
 272           result: 'success'
 273
 274 # Macro: integration-openstack-controller-mininet
 275 # Operation: this macro will spin the controller and mininet vms
 276 # Used by: {project}-csit-* job templates
 277
 278 - wrapper:
 279     name: integration-openstack-controller-mininet
 280     wrappers:
 281       - openstack:
 282           instances:
 283             - cloud-name: 'ODLRPC'
 284               template-name: '{controller-image}'
 285               count: '{controller-vms}'
 286             - cloud-name: 'ODLRPC'
 287               template-name: '{mininet-image}'
 288               count: '{mininet-vms}'
 289
 290 # Macro: integration-openstack-controller-devstack
 291 # Operation: this macro will spin the controller and devstack vms
 292 # Used by: {project}-csit-openstack job templates
 293 - wrapper:
 294     name: integration-openstack-controller-devstack
 295     wrappers:
 296       - openstack:
 297           instances:
 298             - cloud-name: 'ODLRPC'
 299               template-name: '{controller-image}'
 300               count: '{controller-vms}'
 301             - cloud-name: 'ODLRPC'
 302               template-name: '{openstack-image}'
 303               count: '{openstack-vms}'
 304
 305 #################
 306 # Shell Scripts #
 307 #################
 308
 309 # Macro: integration-get-apex-addresses
 310 # Operation: this macro gets the IP addresses of the dynamic vms from apex snapshots
 311 # Used by: {project}-csit-* apex related job templates
 312
 313 - builder:
 314     name: integration-get-apex-addresses
 315     builders:
 316       - shell: !include-raw: integration-get-apex-addresses.sh
 317
 318 # Macro: integration-apex-run-tests
 319 # Operation: this macro just runs the robot suites
 320 # Used by: {project}-csit-* apex related job templates
 321
 322 - builder:
 323     name: integration-apex-run-tests
 324     builders:
 325       - shell: !include-raw: integration-apex-run-tests.sh
 326
 327 # Macro: integration-get-slave-addresses
 328 # Operation: this macro gets the IP addresses of the dynamic vms
 329 # Used by: {project}-csit-* job templates
 330
 331 - builder:
 332     name: integration-get-slave-addresses
 333     builders:
 334       - shell: !include-raw: integration-get-slave-addresses.sh
 335
 336 - builder:
 337     name: integration-set-variables
 338     # Operation: Construct variables not requiring online detection and export using set_variables.env
 339     # Required bash variables:
 340     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 341     builders:
 342       - shell: !include-raw: integration-set-variables.sh
 343       - inject:
 344           properties-file: set_variables.env
 345
 346 - builder:
 347     name: integration-detect-variables
 348     # Operation: Construct variables including online detection and export using detect_variables.env
 349     #   This calls integration-set-variables internally.
 350     # Required bash variables:
 351     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 352     builders:
 353       - integration-set-variables
 354       - shell: !include-raw: integration-detect-variables.sh
 355       - inject:
 356           properties-file: detect_variables.env
 357
 358 # Macro: integration-deply-controller-run-test
 359 # Operation: this macro deploys single contoller and runs test
 360 # Used by: {project}-csit-1node-* job templates
 361
 362 - builder:
 363     name: integration-deploy-controller-run-test
 364     builders:
 365       - shell: !include-raw: integration-deploy-controller-run-test.sh
 366
 367 # Macro: integration-configure-clustering
 368 # Operation: this macro configures the clustering
 369 # Used by: {project}-csit-3node-* job templates
 370
 371 - builder:
 372     name: integration-configure-clustering
 373     builders:
 374       - shell: !include-raw: integration-configure-clustering.sh
 375
 376 # Macro: integration-start-cluster-run-test
 377 # Operation: this macro starts the 3-node cluster and runs test
 378 # Used by: {project}-csit-3node-* job templates
 379
 380 - builder:
 381     name: integration-start-cluster-run-test
 382     builders:
 383       - shell: !include-raw: integration-start-cluster-run-test.sh
 384
 385 # Macro: integration-install-robotframework
 386 # Operation: Installs robotframework using pip to a virtualenv
 387 # Used by: {project}-csit-* job templates
 388
 389 - builder:
 390     name: integration-install-robotframework
 391     builders:
 392       - shell: !include-raw: integration-install-robotframework.sh
 393
 394 # Macro: integration-cleanup-tmp
 395 # Operation: Cleans up temporary files created by build
 396 # Used by: {project}-csit-* job templates
 397
 398 - builder:
 399     name: integration-cleanup-tmp
 400     builders:
 401       - shell: !include-raw: integration-cleanup-tmp.sh
 402
 403 # Macro: integration-multipatch-builder
 404 # Operation: checks out multiple patches and builds custom distribution
 405 # Used by: integration-multipatch-test-* job templates
 406
 407 - builder:
 408     name: integration-multipatch-builder
 409     builders:
 410       - shell: !include-raw: multipatch-distribution.sh
 411
 412 # Macro: integration-cleanup-workspace
 413 # Operation: Cleans up files possibly left there by the previous build
 414 # Used by: {project}-csit-* job templates
 415
 416 - builder:
 417     name: integration-cleanup-workspace
 418     builders:
 419       - shell: !include-raw: integration-cleanup-workspace.sh
 420
 421 # Macro: integration-rebase-patch
 422 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
 423 # Used by: integration-patch-distribution-* job template
 424
 425 - builder:
 426     name: integration-rebase-gerrit-patch
 427     builders:
 428       - shell: !include-raw: integration-rebase-gerrit-patch.sh
 429
 430 # Macro: integration-install-common-functions
 431 # Operation: Copy the common-functions.sh script to csit nodes
 432 # Used by: {project}-csit-* job templates
 433 - builder:
 434     name: integration-install-common-functions
 435     builders:
 436       - shell: !include-raw:
 437           - copy-common-functions.sh
 438           - common-functions.sh
 439
 440 - builder:
 441     name: integration-deploy-openstack-run-test
 442     builders:
 443       - shell: !include-raw:
 444           - integration-deploy-openstack-run-test.sh
 445
 446 - builder:
 447     name: integration-apex-image-manager
 448     builders:
 449       - shell: !include-raw:
 450           - ../opendaylight-infra-apex-image-management.sh
 451
 452 - builder:
 453     name: integration-compare-distributions
 454     # Operation: will compare an already created distribtion (probably last
 455     # published to nexus) to the current distribution being created.
 456     # The comparison is done to find the list of patches that are new to the
 457     # current distribution.
 458     # Required bash variables:
 459     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
 460     builders:
 461       - shell: !include-raw: integration-compare-distributions.sh
 462
 463 - builder:
 464     name: integration-upload-distribution
 465     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
 466     # Arguments:
 467     #   {dist-pom} path to pom file containing the version
 468     # Required bash variables:
 469     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 470     #     Use distribution-karaf-version macro to customize.
 471     builders:
 472       - config-file-provider:
 473           files:
 474             - file-id: global-settings
 475               variable: GLOBAL_SETTINGS_FILE
 476             - file-id: integration-settings
 477               variable: SETTINGS_FILE
 478       - integration-set-variables
 479       - inject:
 480           properties-content: BUNDLE_POM={dist-pom}
 481       - shell: !include-raw-escape: integration-upload-distribution.sh
 482       - inject:
 483           properties-file: integration-upload-distribution.env
 484
 485 ##############
 486 # Publishers #
 487 ##############
 488
 489 - publisher:
 490     name: integration-csit-publish
 491     publishers:
 492       - robot:
 493           output-path: ''
 494           other-files: '{obj:robot_other_files}'
 495           unstable-threshold: '{robot-unstable-threshold}'
 496           pass-threshold: '{robot-pass-threshold}'
 497           only-critical: false
 498       - integration-csit-archive-build
 499       - integration-csit-run-postscript
 500
 501 - publisher:
 502     name: integration-csit-archive-build
 503     publishers:
 504       - postbuildscript:
 505           builders:
 506             - role: BOTH
 507               build-on:
 508                 - ABORTED
 509                 - FAILURE
 510                 - NOT_BUILT
 511                 - SUCCESS
 512                 - UNSTABLE
 513               build-steps:
 514                 - shell: |
 515                     #!/bin/bash
 516                     echo "Archiving csit artifacts"
 517                     cd $WORKSPACE
 518                     mkdir -p ./archives
 519                     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
 520                         NODE_FOLDER="./archives/odl_${i}"
 521                         mkdir -p ${NODE_FOLDER}
 522                         mv odl${i}_* ${NODE_FOLDER} || true
 523                         mv karaf_${i}_*_threads* ${NODE_FOLDER} || true
 524                         mv *_${i}.png ${NODE_FOLDER} || true
 525                         mv /tmp/odl${i}_* ${NODE_FOLDER} || true
 526                         mv gclogs-${i} ${NODE_FOLDER} || true
 527                     done
 528                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
 529                     unzip -d ./archives robot-plugin.zip
 530                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
 531                     # TODO: Can the following line ever fail?
 532                     find . -type f -name *.hprof -print0 \
 533                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
 534                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
 535           mark-unstable-if-failed: true
 536
 537
 538 - publisher:
 539     name: integration-csit-run-postscript
 540     publishers:
 541       - postbuildscript:
 542           builders:
 543             - role: BOTH
 544               build-on:
 545                 - ABORTED
 546                 - FAILURE
 547                 - NOT_BUILT
 548                 - SUCCESS
 549                 - UNSTABLE
 550               build-steps:
 551                 - config-file-provider:
 552                     files:
 553                       - file-id: 'odl-elastic-cloud'
 554                         target: '${HOME}/.netrc'
 555                 - shell: |
 556                     #!/bin/bash
 557                     #The following script runs any configurable postplan stored in test/csit/postplans.
 558                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
 559                         echo "postplan exists!!!"
 560                         echo "Changing the postplan path..."
 561                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
 562                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
 563                         cat postplan.txt
 564                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
 565                             echo "Executing ${line}..."
 566                             ${line}
 567                         done
 568                     fi
 569                     rm ${HOME}/.netrc
 570           mark-unstable-if-failed: true
 571
 572 - publisher:
 573     name: integration-csit-collect-netvirt-logs
 574     publishers:
 575       - postbuildscript:
 576           builders:
 577             - role: BOTH
 578               build-on:
 579                 - ABORTED
 580                 - FAILURE
 581                 - NOT_BUILT
 582                 - SUCCESS
 583                 - UNSTABLE
 584               build-steps:
 585                 - inject:
 586                     properties-file: 'slave_addresses.txt'
 587                 - shell: |
 588                     #!/bin/bash
 589                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
 590                     collect_netvirt_logs
 591           mark-unstable-if-failed: true