jjb/integration/integration-macros.yaml

   1 ---
   2 # TODO: Make all bash constants more readable.
   3 # e.g.: DISTRIBUTION_BRANCH instead of DISTROBRANCH.
   4
   5 - parameter:
   6     name: integration-csit-parameters
   7     parameters:
   8       - opendaylight-infra-parameters:
   9           os-cloud: "{os-cloud}"
  10           project: "integration/test"
  11           branch: "{branch}"
  12           refspec: "refs/heads/{branch}"
  13           artifacts: "{archive-artifacts}"
  14       # CSIT
  15       - integration-bundle-url:
  16           bundle-url: "{bundle-url}"
  17       - integration-repo-url:
  18           repo-url: "{repo-url}"
  19       - integration-test-options:
  20           test-options: "{robot-options}"
  21       - integration-controller-parameters:
  22           controller-debug-map: "{debug-map}"
  23           controller-features: "{install-features}"
  24           controller-max-mem: "{controller-max-mem}"
  25           controller-scope: "{controller-scope}"
  26           controller-use-features-boot: "{use-features-boot}"
  27           feature-elasticsearch-attribute: "{elasticsearch-attribute}"
  28
  29 - parameter:
  30     name: integration-controller-parameters
  31     parameters:
  32       - string:
  33           name: CONTROLLERSCOPE
  34           default: "{controller-scope}"
  35           description: "Feature Only or Feature All test"
  36
  37       - string:
  38           name: CONTROLLERFEATURES
  39           default: "{controller-features}"
  40           description: |
  41             Features to install in the controller separated by comma.
  42             (note: Do not include decanter* features used for harvesting JVM metrics.
  43              They are inserted in install scripts.)
  44
  45       - string:
  46           name: CONTROLLERDEBUGMAP
  47           default: "{controller-debug-map}"
  48           description: |
  49             Key:Value map of debugs to enable on the controller separated by spaces.
  50             The module should not include org.opendaylight.
  51             The levels must be one of: ERROR, WARN, INFO, DEBUG, TRACE.
  52             (ex. ovsdb.lib:ERROR netvirt.vpnservice:INFO)
  53
  54       - string:
  55           name: CONTROLLERMAXMEM
  56           default: "{controller-max-mem}"
  57           description: |
  58             Max Heap size to use when starting the controller.
  59
  60       - string:
  61           name: ELASTICSEARCHATTRIBUTE
  62           default: "{feature-elasticsearch-attribute}"
  63           description: |
  64             Configure the period the script should harvest JVM metrics.
  65             Possible values: long (=120000)
  66                              short (=5000)
  67                              any number between 5000 and 120000
  68                              disabled (use for disabling harvesting)
  69
  70       - string:
  71           name: USEFEATURESBOOT
  72           default: "{controller-use-features-boot}"
  73           description: "Boolean to toggle whether featuresBoot is appended or ignored"
  74
  75 - parameter:
  76     name: integration-distribution-branch
  77     parameters:
  78       - string:
  79           name: DISTROBRANCH
  80           default: "{branch}"
  81           description: "Distribution GIT branch"
  82
  83 - parameter:
  84     name: distribution-karaf-version
  85     # Arguments:
  86     #   karaf-version: Provide, otherwise defaults.yaml value is used.
  87     parameters:
  88       - string:
  89           name: KARAF_VERSION
  90           default: "{karaf-version}"
  91           description: "karaf3, karaf4, odl, affects ODL deploy scripts"
  92
  93 - parameter:
  94     name: integration-bundle-url
  95     parameters:
  96       - string:
  97           name: BUNDLE_URL
  98           default: "{bundle-url}"
  99           description: "URL to karaf distribution zip"
 100
 101 - parameter:
 102     name: integration-repo-url
 103     parameters:
 104       - string:
 105           name: REPO_URL
 106           default: "{repo-url}"
 107           description: "URL to feature repo artifact"
 108
 109 - parameter:
 110     name: integration-jdk-version
 111     parameters:
 112       - string:
 113           name: JDKVERSION
 114           default: "{jdkversion}"
 115           description: "Parameter to indicate JAVA Version"
 116
 117 - parameter:
 118     name: integration-distribution-stream
 119     parameters:
 120       - string:
 121           name: DISTROSTREAM
 122           default: "{stream}"
 123           description: "Distribution stream string, for suites to know which behavior to expect"
 124
 125 - parameter:
 126     name: integration-script-plan
 127     parameters:
 128       - string:
 129           name: SCRIPTPLAN
 130           default: "{script-plan}"
 131           description: "Script plan we will run before downloading and un-archiving am ODL distribution"
 132
 133 - parameter:
 134     name: integration-config-plan
 135     parameters:
 136       - string:
 137           name: CONFIGPLAN
 138           default: "{config-plan}"
 139           description: "Config plan we will run after un-archiving and starting an ODL distribution"
 140
 141 - parameter:
 142     name: integration-stream-test-plan
 143     parameters:
 144       - string:
 145           name: STREAMTESTPLAN
 146           default: "{stream-test-plan}"
 147           description: "Stream-specific test plan we will run"
 148
 149 - parameter:
 150     name: integration-test-plan
 151     parameters:
 152       - string:
 153           name: TESTPLAN
 154           default: "{test-plan}"
 155           description: "General test plan we will run unless stream-specific one is found"
 156
 157 - parameter:
 158     name: integration-test-suites
 159     parameters:
 160       - string:
 161           name: SUITES
 162           default: "{test-suites}"
 163           description: |
 164             List of space-separated suites. Useful when running specific suites within a testplan.
 165             Do not include ${{WORKSPACE}}/test/csit/suites/ in the values:
 166             (ex. openstack/connectivity/l2.robot openstack/clustering/ha_l2.robot
 167              openstack/tempest/tempest.robot netvirt/elan/elan.robot).
 168
 169 - parameter:
 170     name: integration-test-options
 171     parameters:
 172       - string:
 173           name: TESTOPTIONS
 174           default: "{test-options}"
 175           description: "Robot command options"
 176
 177 - parameter:
 178     name: integration-distribution-git-url
 179     parameters:
 180       - string:
 181           name: DISTROGITURL
 182           default: "{git-url}/integration/distribution"
 183           description: "Distribution GIT URL (do not modify)"
 184
 185 - parameter:
 186     name: integration-patch-refspec
 187     parameters:
 188       - string:
 189           name: PATCHREFSPEC
 190           default: "{branch}"
 191           description: "Integration Patch Refspec"
 192
 193 # Macro: integration-gerrit-scm
 194 # Operation: this macro downloads a project gerrit
 195 # Used by: all csit jobs
 196
 197 - scm:
 198     name: integration-gerrit-scm
 199     scm:
 200       - git:
 201           credentials-id: "opendaylight-jenkins-ssh"
 202           url: "$GIT_BASE"
 203           basedir: "{basedir}"
 204           refspec: "{refspec}"
 205           branches:
 206             - "origin/{branch}"
 207           skip-tag: true
 208           choosing-strategy: "gerrit"
 209
 210 # Macro: integration-distribution-scm
 211 # Operation: this macro downloads the integration/distribution repo using distribution as basedir
 212 # Used by: integration-patch-test-{stream} job template
 213
 214 - scm:
 215     name: integration-distribution-scm
 216     scm:
 217       - git:
 218           credentials-id: "opendaylight-jenkins-ssh"
 219           basedir: "distribution"
 220           url: "$DISTROGITURL"
 221           refspec: ""
 222           branches:
 223             - "origin/{branch}"
 224           skip-tag: true
 225
 226 # Macro: integration-trigger-patch-submitted
 227 # This is a single macro to use for all csit jobs who vote on every (relevant) patch set.
 228 # Only 'recheck' trigger word is supported, it always triggers the full set of relevant jobs,
 229 # in order to prevent Jenkins from starting only a subset and still voting Verified+1.
 230 # Contrary to gerrit-trigger-patch-submitted, this votes +1 also on unstable runs.
 231 # Arguments:
 232 #     gerrit-server-name: name of gerrit server to listen to
 233 #     project: pattern to match triggering projects
 234 #     branch: triggering branch name
 235 #     files: pattern to match triggering filepaths
 236 - trigger:
 237     name: integration-trigger-patch-submitted
 238     triggers:
 239       - gerrit:
 240           server-name: "{gerrit-server-name}"
 241           trigger-on:
 242             - patchset-created-event:
 243                 exclude-drafts: true
 244                 exclude-trivial-rebase: false
 245                 exclude-no-code-change: false
 246             - draft-published-event
 247             - comment-added-contains-event:
 248                 comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 249           override-votes: true
 250           gerrit-build-unstable-verified-value: +1
 251           gerrit-build-unstable-codereview-value: 0
 252           projects:
 253             - project-compare-type: ANT
 254               project-pattern: "{project}"
 255               branches:
 256                 - branch-compare-type: ANT
 257                   branch-pattern: "**/{branch}"
 258               file-paths:
 259                 - compare-type: ANT
 260                   pattern: "{files}"
 261
 262 # Macro: integration-csit-only
 263 # Operation: this macro sets a trigger on reverse job list
 264 # Used by: csit -only- job templates
 265
 266 - trigger:
 267     name: integration-csit-trigger
 268     triggers:
 269       - reverse:
 270           jobs: "{jobs}"
 271           result: "success"
 272
 273 # Macro: integration-openstack-controller-mininet
 274 # Operation: this macro will spin the controller and mininet vms
 275 # Used by: {project}-csit-* job templates
 276
 277 - wrapper:
 278     name: integration-openstack-controller-mininet
 279     wrappers:
 280       - openstack:
 281           instances:
 282             - cloud-name: "ODLRPC"
 283               template-name: "{controller-image}"
 284               count: "{controller-vms}"
 285             - cloud-name: "ODLRPC"
 286               template-name: "{mininet-image}"
 287               count: "{mininet-vms}"
 288
 289 # Macro: integration-openstack-controller-devstack
 290 # Operation: this macro will spin the controller and devstack vms
 291 # Used by: {project}-csit-openstack job templates
 292 - wrapper:
 293     name: integration-openstack-controller-devstack
 294     wrappers:
 295       - openstack:
 296           instances:
 297             - cloud-name: "ODLRPC"
 298               template-name: "{controller-image}"
 299               count: "{controller-vms}"
 300             - cloud-name: "ODLRPC"
 301               template-name: "{openstack-image}"
 302               count: "{openstack-vms}"
 303
 304 #################
 305 # Shell Scripts #
 306 #################
 307
 308 # Macro: integration-get-apex-addresses
 309 # Operation: this macro gets the IP addresses of the dynamic vms from apex snapshots
 310 # Used by: {project}-csit-* apex related job templates
 311
 312 - builder:
 313     name: integration-get-apex-addresses
 314     builders:
 315       - shell: !include-raw: integration-get-apex-addresses.sh
 316
 317 # Macro: integration-apex-run-tests
 318 # Operation: this macro just runs the robot suites
 319 # Used by: {project}-csit-* apex related job templates
 320
 321 - builder:
 322     name: integration-apex-run-tests
 323     builders:
 324       - shell: !include-raw: integration-apex-run-tests.sh
 325
 326 # Macro: integration-get-slave-addresses
 327 # Operation: this macro gets the IP addresses of the dynamic vms
 328 # Used by: {project}-csit-* job templates
 329
 330 - builder:
 331     name: integration-get-slave-addresses
 332     builders:
 333       - shell: !include-raw: integration-get-slave-addresses.sh
 334
 335 - builder:
 336     name: integration-set-variables
 337     # Operation: Construct variables not requiring online detection and export using set_variables.env
 338     # Required bash variables:
 339     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 340     builders:
 341       - shell: !include-raw: integration-set-variables.sh
 342       - inject:
 343           properties-file: set_variables.env
 344
 345 - builder:
 346     name: integration-detect-variables
 347     # Operation: Construct variables including online detection and export using detect_variables.env
 348     #   This calls integration-set-variables internally.
 349     # Required bash variables:
 350     #   KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set.
 351     builders:
 352       - integration-set-variables
 353       - shell: !include-raw: integration-detect-variables.sh
 354       - inject:
 355           properties-file: detect_variables.env
 356
 357 # Macro: integration-deply-controller-run-test
 358 # Operation: this macro deploys single contoller and runs test
 359 # Used by: {project}-csit-1node-* job templates
 360
 361 - builder:
 362     name: integration-deploy-controller-run-test
 363     builders:
 364       - shell: !include-raw: integration-deploy-controller-run-test.sh
 365
 366 # Macro: integration-configure-clustering
 367 # Operation: this macro configures the clustering
 368 # Used by: {project}-csit-3node-* job templates
 369
 370 - builder:
 371     name: integration-configure-clustering
 372     builders:
 373       - shell: !include-raw: integration-configure-clustering.sh
 374
 375 # Macro: integration-start-cluster-run-test
 376 # Operation: this macro starts the 3-node cluster and runs test
 377 # Used by: {project}-csit-3node-* job templates
 378
 379 - builder:
 380     name: integration-start-cluster-run-test
 381     builders:
 382       - shell: !include-raw: integration-start-cluster-run-test.sh
 383
 384 # Macro: integration-install-robotframework
 385 # Operation: Installs robotframework using pip to a virtualenv
 386 # Used by: {project}-csit-* job templates
 387
 388 - builder:
 389     name: integration-install-robotframework
 390     builders:
 391       - shell: !include-raw: integration-install-robotframework.sh
 392
 393 # Macro: integration-cleanup-tmp
 394 # Operation: Cleans up temporary files created by build
 395 # Used by: {project}-csit-* job templates
 396
 397 - builder:
 398     name: integration-cleanup-tmp
 399     builders:
 400       - shell: !include-raw: integration-cleanup-tmp.sh
 401
 402 # Macro: integration-multipatch-builder
 403 # Operation: checks out multiple patches and builds custom distribution
 404 # Used by: integration-multipatch-test-* job templates
 405
 406 - builder:
 407     name: integration-multipatch-builder
 408     builders:
 409       - shell: !include-raw: multipatch-distribution.sh
 410
 411 # Macro: integration-cleanup-workspace
 412 # Operation: Cleans up files possibly left there by the previous build
 413 # Used by: {project}-csit-* job templates
 414
 415 - builder:
 416     name: integration-cleanup-workspace
 417     builders:
 418       - shell: !include-raw: integration-cleanup-workspace.sh
 419
 420 # Macro: integration-rebase-patch
 421 # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH.
 422 # Used by: integration-patch-distribution-* job template
 423
 424 - builder:
 425     name: integration-rebase-gerrit-patch
 426     builders:
 427       - shell: !include-raw: integration-rebase-gerrit-patch.sh
 428
 429 # Macro: integration-install-common-functions
 430 # Operation: Copy the common-functions.sh script to csit nodes
 431 # Used by: {project}-csit-* job templates
 432 - builder:
 433     name: integration-install-common-functions
 434     builders:
 435       - shell: !include-raw:
 436           - copy-common-functions.sh
 437           - common-functions.sh
 438
 439 - builder:
 440     name: integration-deploy-openstack-run-test
 441     builders:
 442       - shell: !include-raw:
 443           - integration-deploy-openstack-run-test.sh
 444
 445 - builder:
 446     name: integration-apex-image-manager
 447     builders:
 448       - shell: !include-raw:
 449           - ../opendaylight-infra-apex-image-management.sh
 450
 451 - builder:
 452     name: integration-compare-distributions
 453     # Operation: will compare an already created distribtion (probably last
 454     # published to nexus) to the current distribution being created.
 455     # The comparison is done to find the list of patches that are new to the
 456     # current distribution.
 457     # Required bash variables:
 458     #   KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables.
 459     builders:
 460       - shell: !include-raw: integration-compare-distributions.sh
 461
 462 - builder:
 463     name: integration-upload-distribution
 464     # Upload distribution karaf artifact to Nexus SNAPSHOT repo for testing.
 465     # Arguments:
 466     #   {dist-pom} path to pom file containing the version
 467     # Required bash variables:
 468     #   KARAF_VERSION: 'karaf4' or 'karaf3'.
 469     #     Use distribution-karaf-version macro to customize.
 470     builders:
 471       - config-file-provider:
 472           files:
 473             - file-id: global-settings
 474               variable: GLOBAL_SETTINGS_FILE
 475             - file-id: integration-settings
 476               variable: SETTINGS_FILE
 477       - integration-set-variables
 478       - inject:
 479           properties-content: BUNDLE_POM={dist-pom}
 480       - shell: !include-raw-escape: integration-upload-distribution.sh
 481       - inject:
 482           properties-file: integration-upload-distribution.env
 483
 484 ##############
 485 # Publishers #
 486 ##############
 487
 488 - publisher:
 489     name: integration-csit-publish
 490     publishers:
 491       - robot:
 492           output-path: ""
 493           other-files: "{obj:robot_other_files}"
 494           unstable-threshold: "{robot-unstable-threshold}"
 495           pass-threshold: "{robot-pass-threshold}"
 496           only-critical: false
 497       - integration-csit-archive-build
 498       - integration-csit-run-postscript
 499
 500 - publisher:
 501     name: integration-csit-archive-build
 502     publishers:
 503       - postbuildscript:
 504           builders:
 505             - role: BOTH
 506               build-on:
 507                 - ABORTED
 508                 - FAILURE
 509                 - NOT_BUILT
 510                 - SUCCESS
 511                 - UNSTABLE
 512               build-steps:
 513                 - shell: |
 514                     #!/bin/bash
 515                     echo "Archiving csit artifacts"
 516                     cd $WORKSPACE
 517                     mkdir -p ./archives
 518                     for i in `seq 1 ${NUM_ODL_SYSTEM}`; do
 519                         NODE_FOLDER="./archives/odl_${i}"
 520                         mkdir -p ${NODE_FOLDER}
 521                         mv odl${i}_* ${NODE_FOLDER} || true
 522                         mv karaf_${i}_*_threads* ${NODE_FOLDER} || true
 523                         mv *_${i}.png ${NODE_FOLDER} || true
 524                         mv /tmp/odl${i}_* ${NODE_FOLDER} || true
 525                         mv gclogs-${i} ${NODE_FOLDER} || true
 526                     done
 527                     curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip"
 528                     unzip -d ./archives robot-plugin.zip
 529                     mv *.log *.log.gz *.csv *.png ./archives || true  # Don't fail if file missing
 530                     # TODO: Can the following line ever fail?
 531                     find . -type f -name *.hprof -print0 \
 532                         | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz
 533                     # TODO: Tweak the compression level if better ratio (or speed) is needed.
 534           mark-unstable-if-failed: true
 535
 536 - publisher:
 537     name: integration-csit-run-postscript
 538     publishers:
 539       - postbuildscript:
 540           builders:
 541             - role: BOTH
 542               build-on:
 543                 - ABORTED
 544                 - FAILURE
 545                 - NOT_BUILT
 546                 - SUCCESS
 547                 - UNSTABLE
 548               build-steps:
 549                 - config-file-provider:
 550                     files:
 551                       - file-id: "odl-elastic-cloud"
 552                         target: "${HOME}/.netrc"
 553                 - shell: |
 554                     #!/bin/bash
 555                     #The following script runs any configurable postplan stored in test/csit/postplans.
 556                     if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then
 557                         echo "postplan exists!!!"
 558                         echo "Changing the postplan path..."
 559                         script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN}
 560                         cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt
 561                         cat postplan.txt
 562                         egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do
 563                             echo "Executing ${line}..."
 564                             ${line}
 565                         done
 566                     fi
 567                     rm ${HOME}/.netrc
 568           mark-unstable-if-failed: true
 569
 570 - publisher:
 571     name: integration-csit-collect-netvirt-logs
 572     publishers:
 573       - postbuildscript:
 574           builders:
 575             - role: BOTH
 576               build-on:
 577                 - ABORTED
 578                 - FAILURE
 579                 - NOT_BUILT
 580                 - SUCCESS
 581                 - UNSTABLE
 582               build-steps:
 583                 - inject:
 584                     properties-file: "slave_addresses.txt"
 585                 - shell: |
 586                     #!/bin/bash
 587                     source /tmp/common-functions.sh ${BUNDLEFOLDER}
 588                     collect_netvirt_logs
 589           mark-unstable-if-failed: true