jjb/integration/integration-macros.yaml

   1 ---
   2 # TODO: Make all bash constants more readable.
   3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
   4
   5 - parameter:
   6     name: integration-csit-parameters
   7     parameters:
   8       - opendaylight-infra-parameters:
   9           os-cloud: "{os-cloud}"
  10           project: "integration/test"
  11           branch: "{branch}"
  12           refspec: "refs/heads/{branch}"
  13           artifacts: "{archive-artifacts}"
  14       # CSIT
  15       - integration-bundle-url:
  16           bundle-url: "{bundle-url}"
  17       - integration-repo-url:
  18           repo-url: "{repo-url}"
  19       - integration-test-options:
  20           test-options: "{robot-options}"
  21       - integration-controller-parameters:
  22           controller-debug-map: "{debug-map}"
  23           controller-features: "{install-features}"
  24           controller-max-mem: "{controller-max-mem}"
  25           controller-scope: "{controller-scope}"
  26           controller-use-features-boot: "{use-features-boot}"
  27           feature-elasticsearch-attribute: "{elasticsearch-attribute}"
  28
  29 - parameter:
  30     name: integration-controller-parameters
  31     parameters:
  32       - string:
  33           name: CONTROLLERSCOPE
  34           default: "{controller-scope}"
  35           description: "Feature Only or Feature All test"
  36
  37       - string:
  38           name: CONTROLLERFEATURES
  39           default: "{controller-features}"
  40           description: |
  41             Features to install in the controller separated by comma.
  42             (note: Do not include decanter* features used for harvesting JVM metrics.
  43              They are inserted in install scripts.)
  44
  45       - string:
  46           name: CONTROLLERDEBUGMAP
  47           default: "{controller-debug-map}"
  48           description: |
  49             Key:Value map of debugs to enable on the controller separated by spaces.
  50             The module should not include org.opendaylight.
  51             The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
  52             (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
  53
  54       - string:
  55           name: CONTROLLERMAXMEM
  56           default: "{controller-max-mem}"
  57           description: |
  58             Max Heap size to use when starting the controller.
  59
  60       - string:
  61           name: ELASTICSEARCHATTRIBUTE
  62           default: "{feature-elasticsearch-attribute}"
  63           description: |
  64             Configure the period the script should harvest JVM metrics.
  65             Possible values: long (=120000)
  66                              short (=5000)
  67                              any number between 5000 and 120000
  68                              disabled (use for disabling harvesting)
  69
  70       - string:
  71           name: USEFEATURESBOOT
  72           default: "{controller-use-features-boot}"
  73           description: "Boolean to toggle whether featuresBoot is appended or ignored"
  74
  75 - parameter:
  76     name: integration-distribution-branch
  77     parameters:
  78       - string:
  79           name: DISTROBRANCH
  80           default: "{branch}"
  81           description: "Distribution GIT branch"
  82
  83 - parameter:
  84     name: distribution-karaf-version
  85     # Arguments:
  86     #   karaf-version: Provide, otherwise defaults.yaml value is used.
  87     parameters:
  88       - string:
  89           name: KARAF_VERSION
  90           default: "{karaf-version}"
  91           description: "karaf3, karaf4, odl, affects ODL deploy scripts"
  92
  93 - parameter:
  94     name: integration-bundle-url
  95     parameters:
  96       - string:
  97           name: BUNDLE_URL
  98           default: "{bundle-url}"
  99           description: "URL to karaf distribution zip"
 100
 101 - parameter:
 102     name: integration-repo-url
 103     parameters:
 104       - string:
 105           name: REPO_URL
 106           default: "{repo-url}"
 107           description: "URL to feature repo artifact"
 108
 109 - parameter:
 110     name: integration-jdk-version
 111     parameters:
 112       - string:
 113           name: JDKVERSION
 114           default: "{jdkversion}"
 115           description: "Parameter to indicate JAVA Version"
 116
 117 - parameter:
 118     name: integration-distribution-stream
 119     parameters:
 120       - string:
 121           name: DISTROSTREAM
 122           default: "{stream}"
 123           description: "Distribution stream string, for suites to know which behavior to expect"
 124
 125 - parameter:
 126     name: integration-script-plan
 127     parameters:
 128       - string:
 129           name: SCRIPTPLAN
 130           default: "{script-plan}"
 131           description: "Script plan we will run before downloading and un-archiving am ODL distribution"
 132
 133 - parameter:
 134     name: integration-config-plan
 135     parameters:
 136       - string:
 137           name: CONFIGPLAN
 138           default: "{config-plan}"
 139           description: "Config plan we will run after un-archiving and starting an ODL distribution"
 140
 141 - parameter:
 142     name: integration-stream-test-plan
 143     parameters:
 144       - string:
 145           name: STREAMTESTPLAN
 146           default: "{stream-test-plan}"
 147           description: "Stream-specific test plan we will run"
 148
 149 - parameter:
 150     name: integration-test-plan
 151     parameters:
 152       - string:
 153           name: TESTPLAN
 154           default: "{test-plan}"
 155           description: "General test plan we will run unless stream-specific one is found"
 156
 157 - parameter:
 158     name: integration-test-suites
 159     parameters:
 160       - string:
 161           name: SUITES
 162           default: "{test-suites}"
 163           description: |
 164             List of space-separated suites. Useful when running specific suites within a testplan.
 165             Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
 166             (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
 167              openstack/tempest/tempest.robot netvirt/elan/elan.robot).
 168
 169 - parameter:
 170     name: integration-test-options
 171     parameters:
 172       - string:
 173           name: TESTOPTIONS
 174           default: "{test-options}"
 175           description: "Robot command options"
 176
 177 - parameter:
 178     name: integration-distribution-git-url
 179     parameters:
 180       - string:
 181           name: DISTROGITURL
 182           default: "{git-url}/integration/distribution"
 183           description: "Distribution GIT URL (do not modify)"
 184
 185 - parameter:
 186     name: integration-patch-refspec
 187     parameters:
 188       - string:
 189           name: PATCHREFSPEC
 190           default: "{branch}"
 191           description: "Integration Patch Refspec"
 192
 193 # Macro: integration-gerrit-scm
 194 # Operation: this macro downloads a project gerrit
 195 # Used by: all csit jobs
 196
 197 - scm:
 198     name: integration-gerrit-scm
 199     scm:
 200       - git:
 201           credentials-id: "opendaylight-jenkins-ssh"
 202           url: "$GIT_BASE"
 203           basedir: "{basedir}"
 204           refspec: "{refspec}"
 205           branches:
 206             - "origin/{branch}"
 207           skip-tag: true
 208           choosing-strategy: "gerrit"
 209
 210 # Macro: integration-distribution-scm
 211 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
 212 # Used by: integration-patch-test-{stream} job template
 213
 214 - scm:
 215     name: integration-distribution-scm
 216     scm:
 217       - git:
 218           credentials-id: "opendaylight-jenkins-ssh"
 219           basedir: "distribution"
 220           url: "$DISTROGITURL"
 221           refspec: ""
 222           branches:
 223             - "origin/{branch}"
 224           skip-tag: true
 225
 226 # Macro: integration-trigger-patch-submitted
 227 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
 228 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
 229 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
 230 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
 231 # Arguments:
 232 #     gerrit-server-name: name of gerrit server to listen to
 233 #     project: pattern to match triggering projects
 234 #     branch: triggering branch name
 235 #     files: pattern to match triggering filepaths
 236 - trigger:
 237     name: integration-trigger-patch-submitted
 238     triggers:
 239       - gerrit:
 240           server-name: "{gerrit-server-name}"
 241           trigger-on:
 242             - patchset-created-event:
 243                 exclude-drafts: true
 244                 exclude-trivial-rebase: false
 245                 exclude-no-code-change: false
 246             - draft-published-event
 247             - comment-added-contains-event:
 248                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 249           override-votes: true
 250           gerrit-build-unstable-verified-value: +1
 251           gerrit-build-unstable-codereview-value: 0
 252           projects:
 253             - project-compare-type: ANT
 254               project-pattern: "{project}"
 255               branches:
 256                 - branch-compare-type: ANT
 257                   branch-pattern: "**/{branch}"
 258               file-paths:
 259                 - compare-type: ANT
 260                   pattern: "{files}"
 261
 262 # Macro: integration-csit-only
 263 # Operation: this macro sets a trigger on reverse job list
 264 # Used by: csit -only- job templates
 265
 266 - trigger:
 267     name: integration-csit-trigger
 268     triggers:
 269       - reverse:
 270           jobs: "{jobs}"
 271           result: "success"
 272
 273 # Macro: integration-openstack-controller-mininet
 274 # Operation: this macro will spin the controller and mininet vms
 275 # Used by: {project}-csit-* job templates
 276
 277 - wrapper:
 278     name: integration-openstack-controller-mininet
 279     wrappers:
 280       - openstack:
 281           instances:
 282             - cloud-name: "ODLRPC"
 283               template-name: "{controller-image}"
 284               count: "{controller-vms}"
 285             - cloud-name: "ODLRPC"
 286               template-name: "{mininet-image}"
 287               count: "{mininet-vms}"
 288
 289 # Macro: integration-openstack-controller-devstack
 290 # Operation: this macro will spin the controller and devstack vms
 291 # Used by: {project}-csit-openstack job templates
 292 - wrapper:
 293     name: integration-openstack-controller-devstack
 294     wrappers:
 295       - openstack:
 296           instances:
 297             - cloud-name: "ODLRPC"
 298               template-name: "{controller-image}"
 299               count: "{controller-vms}"
 300             - cloud-name: "ODLRPC"
 301               template-name: "{openstack-image}"
 302               count: "{openstack-vms}"
 303
 304 #################
 305 # Shell Scripts #
 306 #################
 307
 308 # Macro: integration-get-slave-addresses
 309 # Operation: this macro gets the IP addresses of the dynamic vms
 310 # Used by: {project}-csit-* job templates
 311
 312 - builder:
 313     name: integration-get-slave-addresses
 314     builders:
 315       - shell: !include-raw: integration-get-slave-addresses.sh
 316
 317 - builder:
 318     name: integration-set-variables
 319     # Operation: Construct variables not requiring online detection and export using set_variables.env
 320     # Required bash variables:
 321     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 322     builders:
 323       - shell: !include-raw: integration-set-variables.sh
 324       - inject:
 325           properties-file: set_variables.env
 326
 327 - builder:
 328     name: integration-detect-variables
 329     # Operation: Construct variables including online detection and export using detect_variables.env
 330     #   This calls integration-set-variables internally.
 331     # Required bash variables:
 332     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 333     builders:
 334       - integration-set-variables
 335       - shell: !include-raw: integration-detect-variables.sh
 336       - inject:
 337           properties-file: detect_variables.env
 338
 339 # Macro: integration-deply-controller-run-test
 340 # Operation: this macro deploys single contoller and runs test
 341 # Used by: {project}-csit-1node-* job templates
 342
 343 - builder:
 344     name: integration-deploy-controller-run-test
 345     builders:
 346       - shell: !include-raw: integration-deploy-controller-run-test.sh
 347
 348 # Macro: integration-configure-clustering
 349 # Operation: this macro configures the clustering
 350 # Used by: {project}-csit-3node-* job templates
 351
 352 - builder:
 353     name: integration-configure-clustering
 354     builders:
 355       - shell: !include-raw: integration-configure-clustering.sh
 356
 357 # Macro: integration-start-cluster-run-test
 358 # Operation: this macro starts the 3-node cluster and runs test
 359 # Used by: {project}-csit-3node-* job templates
 360
 361 - builder:
 362     name: integration-start-cluster-run-test
 363     builders:
 364       - shell: !include-raw: integration-start-cluster-run-test.sh
 365
 366 # Macro: integration-install-robotframework
 367 # Operation: Installs robotframework using pip to a virtualenv
 368 # Used by: {project}-csit-* job templates
 369
 370 - builder:
 371     name: integration-install-robotframework
 372     builders:
 373       - shell: !include-raw: integration-install-robotframework.sh
 374
 375 # Macro: integration-cleanup-tmp
 376 # Operation: Cleans up temporary files created by build
 377 # Used by: {project}-csit-* job templates
 378
 379 - builder:
 380     name: integration-cleanup-tmp
 381     builders:
 382       - shell: !include-raw: integration-cleanup-tmp.sh
 383
 384 # Macro: integration-multipatch-builder
 385 # Operation: checks out multiple patches and builds custom distribution
 386 # Used by: integration-multipatch-test-* job templates
 387
 388 - builder:
 389     name: integration-multipatch-builder
 390     builders:
 391       - shell: !include-raw: multipatch-distribution.sh
 392
 393 # Macro: integration-cleanup-workspace
 394 # Operation: Cleans up files possibly left there by the previous build
 395 # Used by: {project}-csit-* job templates
 396
 397 - builder:
 398     name: integration-cleanup-workspace
 399     builders:
 400       - shell: !include-raw: integration-cleanup-workspace.sh
 401
 402 # Macro: integration-rebase-patch
 403 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
 404 # Used by: integration-patch-distribution-* job template
 405
 406 - builder:
 407     name: integration-rebase-gerrit-patch
 408     builders:
 409       - shell: !include-raw: integration-rebase-gerrit-patch.sh
 410
 411 # Macro: integration-install-common-functions
 412 # Operation: Copy the common-functions.sh script to csit nodes
 413 # Used by: {project}-csit-* job templates
 414 - builder:
 415     name: integration-install-common-functions
 416     builders:
 417       - shell: !include-raw:
 418           - copy-common-functions.sh
 419           - common-functions.sh
 420
 421 - builder:
 422     name: integration-deploy-openstack-run-test
 423     builders:
 424       - shell: !include-raw:
 425           - integration-deploy-openstack-run-test.sh
 426
 427 - builder:
 428     name: integration-compare-distributions
 429     # Operation: will compare an already created distribtion (probably last
 430     # published to nexus) to the current distribution being created.
 431     # The comparison is done to find the list of patches that are new to the
 432     # current distribution.
 433     # Required bash variables:
 434     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
 435     builders:
 436       - shell: !include-raw: integration-compare-distributions.sh
 437
 438 - builder:
 439     name: integration-upload-distribution
 440     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
 441     # Arguments:
 442     #   {dist-pom} path to pom file containing the version
 443     # Required bash variables:
 444     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 445     #     Use distribution-karaf-version macro to customize.
 446     builders:
 447       - config-file-provider:
 448           files:
 449             - file-id: global-settings
 450               variable: GLOBAL_SETTINGS_FILE
 451             - file-id: integration-settings
 452               variable: SETTINGS_FILE
 453       - integration-set-variables
 454       - inject:
 455           properties-content: BUNDLE_POM={dist-pom}
 456       - shell: !include-raw-escape: integration-upload-distribution.sh
 457       - inject:
 458           properties-file: integration-upload-distribution.env
 459
 460 ##############
 461 # Publishers #
 462 ##############
 463
 464 - publisher:
 465     name: integration-csit-publish
 466     publishers:
 467       - robot:
 468           output-path: ""
 469           other-files: "{obj:robot_other_files}"
 470           unstable-threshold: "{robot-unstable-threshold}"
 471           pass-threshold: "{robot-pass-threshold}"
 472           only-critical: false
 473       - integration-csit-archive-build
 474       - integration-csit-run-postscript
 475
 476 - publisher:
 477     name: integration-csit-archive-build
 478     publishers:
 479       - postbuildscript:
 480           builders:
 481             - role: BOTH
 482               build-on:
 483                 - ABORTED
 484                 - FAILURE
 485                 - NOT_BUILT
 486                 - SUCCESS
 487                 - UNSTABLE
 488               build-steps:
 489                 - shell: |
 490                     #!/bin/bash
 491                     echo "Archiving csit artifacts"
 492                     cd $WORKSPACE
 493                     mkdir -p ./archives
 494                     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
 495                         NODE_FOLDER="./archives/odl_${i}"
 496                         mkdir -p ${NODE_FOLDER}
 497                         mv odl${i}_* ${NODE_FOLDER} || true
 498                         mv karaf_${i}_*_threads* ${NODE_FOLDER} || true
 499                         mv *_${i}.png ${NODE_FOLDER} || true
 500                         mv /tmp/odl${i}_* ${NODE_FOLDER} || true
 501                         mv gclogs-${i} ${NODE_FOLDER} || true
 502                     done
 503                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
 504                     unzip -d ./archives robot-plugin.zip
 505                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
 506                     # TODO: Can the following line ever fail?
 507                     find . -type f -name *.hprof -print0 \
 508                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
 509                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
 510           mark-unstable-if-failed: true
 511
 512 - publisher:
 513     name: integration-csit-run-postscript
 514     publishers:
 515       - postbuildscript:
 516           builders:
 517             - role: BOTH
 518               build-on:
 519                 - ABORTED
 520                 - FAILURE
 521                 - NOT_BUILT
 522                 - SUCCESS
 523                 - UNSTABLE
 524               build-steps:
 525                 - config-file-provider:
 526                     files:
 527                       - file-id: "odl-elastic-cloud"
 528                         target: "${HOME}/.netrc"
 529                 - shell: |
 530                     #!/bin/bash
 531                     #The following script runs any configurable postplan stored in test/csit/postplans.
 532                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
 533                         echo "postplan exists!!!"
 534                         echo "Changing the postplan path..."
 535                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
 536                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
 537                         cat postplan.txt
 538                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
 539                             echo "Executing ${line}..."
 540                             ${line}
 541                         done
 542                     fi
 543                     rm ${HOME}/.netrc
 544           mark-unstable-if-failed: true
 545
 546 - publisher:
 547     name: integration-csit-collect-netvirt-logs
 548     publishers:
 549       - postbuildscript:
 550           builders:
 551             - role: BOTH
 552               build-on:
 553                 - ABORTED
 554                 - FAILURE
 555                 - NOT_BUILT
 556                 - SUCCESS
 557                 - UNSTABLE
 558               build-steps:
 559                 - inject:
 560                     properties-file: "slave_addresses.txt"
 561                 - shell: |
 562                     #!/bin/bash
 563                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
 564                     collect_netvirt_logs
 565           mark-unstable-if-failed: true