jjb/integration/integration-test-jobs.yaml

   1 ---
   2 - project:
   3     name: integration-test
   4     project: integration/test
   5     project-name: integration-test
   6     jobs:
   7       - 'integration-patch-test-{stream}'
   8       - 'inttest-multipatch'
   9       - 'integration-merge-dashboard'
  10       - '{project-name}-rtd-jobs':
  11           build-node: centos7-builder-2c-2g
  12           project-pattern: integration/test
  13           rtd-build-url: https://readthedocs.org/api/v2/webhook/odl-integrationtest/32934/
  14           rtd-token: 70766f97941a74ef80b84749ac4799e5ddeee99a
  15       - gerrit-tox-verify:
  16           branch: master
  17           stream: master
  18
  19     # CSIT Lists in defaults.yaml
  20     stream:
  21       - aluminium:
  22           branch: 'master'
  23           mvn-version: 'mvn35'
  24           java-version: 'openjdk11'
  25           csit-list: !include: csit-jobs-aluminium.lst
  26           build-order: >
  27               odlparent yangtools mdsal infrautils controller serviceutils aaa netconf
  28               daexim jsonrpc ovsdb neutron openflowplugin genius lispflowmapping netvirt
  29               bgpcep integration/distribution
  30
  31       - magnesium:
  32           branch: 'stable/magnesium'
  33           mvn-version: 'mvn35'
  34           java-version: 'openjdk11'
  35           csit-list: !include: csit-jobs-magnesium.lst
  36           build-order: >
  37               odlparent yangtools mdsal infrautils controller serviceutils aaa netconf
  38               daexim jsonrpc ovsdb neutron openflowplugin genius lispflowmapping netvirt
  39               bgpcep integration/distribution
  40
  41       - sodium:
  42           branch: 'stable/sodium'
  43           mvn-version: 'mvn35'
  44           csit-list: !include: csit-jobs-sodium.lst
  45           build-order: >
  46               odlparent yangtools mdsal infrautils controller serviceutils aaa netconf
  47               daexim ovsdb neutron openflowplugin coe genius lispflowmapping sfc netvirt
  48               bgpcep integration/distribution
  49
  50       - neon:
  51           branch: 'stable/neon'
  52           mvn-version: 'mvn35'
  53           csit-list: !include: csit-jobs-neon.lst
  54           build-order: >
  55               odlparent yangtools mdsal infrautils controller serviceutils aaa netconf
  56               daexim ovsdb neutron openflowplugin coe genius lispflowmapping sfc netvirt
  57               bgpcep integration/distribution
  58
  59 - project:
  60     name: integration-distribution
  61     project: integration/test
  62     project-name: integration-test
  63     jobs:
  64       - 'integration-distribution-test-{stream}'
  65       - 'integration-distribution-weekly-test-trigger-{stream}'
  66       - 'integration-update-csit-tests-{stream}'
  67
  68     # CSIT Lists in defaults.yaml
  69     stream:
  70       - aluminium:
  71           branch: 'master'
  72           csit-blacklist: >
  73               alto bier coe didm dluxapps groupbasedpolicy l2switch nemo
  74               of-config p4plugin packetcable sfc sxp telemetry tsdr unimgr usc
  75               gate longevity sanity tdd tempest wip
  76               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-pike
  77               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-queens-upstream-stateful-itm-direct
  78               netvirt-csit-3node-1cmb
  79               last-line-is-ignored-so-we-need-something-dummy-here-to-make-sure-the-real-last-line-is-used
  80           csit-list: !include: csit-jobs-aluminium.lst
  81           csit-weekly-list: '{csit-weekly-list-aluminium}'
  82       - magnesium:
  83           branch: 'stable/magnesium'
  84           csit-blacklist: >
  85               alto bier coe didm dluxapps groupbasedpolicy l2switch nemo
  86               of-config p4plugin packetcable sfc sxp telemetry tsdr unimgr usc
  87               gate longevity sanity tdd tempest wip
  88               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-pike
  89               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-queens-upstream-stateful-itm-direct
  90               netvirt-csit-3node-1cmb
  91               last-line-is-ignored-so-we-need-something-dummy-here-to-make-sure-the-real-last-line-is-used
  92           csit-list: !include: csit-jobs-magnesium.lst
  93           csit-weekly-list: '{csit-weekly-list-magnesium}'
  94       - sodium:
  95           branch: 'stable/sodium'
  96           csit-blacklist: >
  97               alto bier didm dluxapps groupbasedpolicy jsonrpc l2switch nemo
  98               of-config p4plugin packetcable sxp telemetry tsdr unimgr usc
  99               gate longevity sanity tdd tempest wip
 100               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-pike
 101               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-queens-upstream-stateful-itm-direct
 102               netvirt-csit-3node-1cmb
 103               last-line-is-ignored-so-we-need-something-dummy-here-to-make-sure-the-real-last-line-is-used
 104           csit-list: !include: csit-jobs-sodium.lst
 105           csit-weekly-list: '{csit-weekly-list-sodium}'
 106       - neon:
 107           branch: 'stable/neon'
 108           csit-blacklist: >
 109               alto bier didm dluxapps groupbasedpolicy jsonrpc l2switch nemo
 110               of-config p4plugin packetcable sxp telemetry tsdr unimgr usc
 111               gate longevity sanity tdd tempest wip
 112               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-pike
 113               netvirt-csit-1node-1cmb-0ctl-0cmp-openstack-queens-upstream-stateful-itm-direct
 114               netvirt-csit-3node-1cmb
 115               last-line-is-ignored-so-we-need-something-dummy-here-to-make-sure-the-real-last-line-is-used
 116           csit-list: !include: csit-jobs-neon.lst
 117           csit-weekly-list: '{csit-weekly-list-neon}'
 118
 119 - project:
 120     name: integration-sanity
 121     project: integration/test
 122     project-name: integration-test
 123     jobs:
 124       - 'integration-sanity-test-{stream}'
 125
 126     stream:
 127       - aluminium:
 128           branch: 'master'
 129           karaf-version: odl
 130           csit-sanity-list: '{csit-sanity-list-aluminium}'
 131       - magnesium:
 132           branch: 'stable/magnesium'
 133           karaf-version: odl
 134           csit-sanity-list: '{csit-sanity-list-magnesium}'
 135       - sodium:
 136           branch: 'stable/sodium'
 137           karaf-version: odl
 138           csit-sanity-list: '{csit-sanity-list-sodium}'
 139       - neon:
 140           branch: 'stable/neon'
 141           karaf-version: odl
 142           csit-sanity-list: '{csit-sanity-list-neon}'
 143
 144 - job-template:
 145     name: 'integration-merge-dashboard'
 146     project-type: freestyle
 147     node: centos7-builder-2c-1g
 148
 149     properties:
 150       - opendaylight-infra-properties:
 151           build-days-to-keep: '{build-days-to-keep}'
 152
 153     parameters:
 154       - opendaylight-infra-parameters:
 155           os-cloud: '{os-cloud}'
 156           project: '{project}'
 157           branch: 'master'
 158           refspec: 'refs/heads/master'
 159           artifacts: '{archive-artifacts}'
 160
 161     scm:
 162       - integration-gerrit-scm:
 163           basedir: 'test'
 164           refspec: '$GERRIT_REFSPEC'
 165           branch: 'master'
 166
 167     wrappers:
 168       - opendaylight-infra-wrappers:
 169           build-timeout: '{build-timeout}'
 170
 171     triggers:
 172       - gerrit-trigger-patch-merged:
 173           gerrit-server-name: '{gerrit-server-name}'
 174           name: '{project}'
 175           branch: 'master'
 176           files: 'dashboard/**'
 177           forbidden-files: ''
 178
 179     builders:
 180       - lf-infra-pre-build
 181       - config-file-provider:
 182           files:
 183             - file-id: odl-elastic-cloud
 184               target: '$HOME/.netrc'
 185       - integration-install-robotframework
 186       # yamllint disable-line rule:line-length
 187       - shell: python3 $WORKSPACE/test/csit/scripts/push_dashboard.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
 188
 189     publishers:
 190       - lf-infra-publish
 191       - email-notification:
 192           email-recipients: '{email-recipients}'
 193           email-prefix: '[{project-name}]'
 194
 195 - job-template:
 196     name: 'integration-distribution-test-{stream}'
 197     # Goal: Verify a distribution through all system test available
 198     # Operation: This template takes a distribution and passes all available system test
 199
 200     project-type: freestyle
 201     node: queue-disttest-2c-1g
 202
 203     properties:
 204       - opendaylight-infra-properties:
 205           build-days-to-keep: '{build-days-to-keep}'
 206
 207     parameters:
 208       - integration-bundle-url:
 209           bundle-url: '{bundle-url}'
 210       - distribution-karaf-version:
 211           karaf-version: '{karaf-version}'
 212
 213     builders:
 214       - lf-infra-pre-build
 215       - trigger-builds:
 216           - project: '{csit-list}'
 217             block: true
 218             predefined-parameters: |
 219               BUNDLE_URL=$BUNDLE_URL
 220               KARAF_VERSION=$KARAF_VERSION
 221
 222     wrappers:
 223       - lf-infra-wrappers:
 224           build-timeout: 1440
 225           jenkins-ssh-credential: '{jenkins-ssh-credential}'
 226
 227     publishers:
 228       - opendaylight-infra-generate-csit-status-report
 229       - email-notification:
 230           email-recipients: '{email-recipients}'
 231           email-prefix: '[integration]'
 232       - lf-infra-publish
 233
 234 - job-template:
 235     name: 'integration-sanity-test-{stream}'
 236     # Goal: Verify a distribution through sanity test
 237     # Operation: This template takes a distribution and passes sanity test
 238     # This jobs works for both managed & unmanaged distro.
 239
 240     project-type: freestyle
 241     node: queue-intque-2c-1g
 242
 243     properties:
 244       - opendaylight-infra-properties:
 245           build-days-to-keep: '{build-days-to-keep}'
 246
 247     parameters:
 248       - integration-bundle-url:
 249           bundle-url: '{bundle-url}'
 250       - distribution-karaf-version:
 251           karaf-version: '{karaf-version}'
 252
 253     triggers:
 254       - timed: '{schedule-daily}'
 255
 256     builders:
 257       - lf-infra-pre-build
 258       - trigger-builds:
 259           - project: '{csit-sanity-list}'
 260             block: true
 261             predefined-parameters: |
 262               BUNDLE_URL=$BUNDLE_URL
 263               KARAF_VERSION=$KARAF_VERSION
 264
 265     publishers:
 266       - email-notification:
 267           email-recipients: '{email-recipients}'
 268           email-prefix: '[integration]'
 269
 270 - job-template:
 271     name: 'integration-patch-test-{stream}'
 272     # Template: integration-patch-test-{stream}
 273     # Goal: Build a patch and run all available system test on a distribution containing the change
 274     # Operation: This job template builds a patch, creates a distribution containing the patch, and
 275     # triggers the distribution test when test-integration is used in gerrit comments
 276
 277     project-type: freestyle
 278     node: centos7-builder-2c-8g
 279
 280     properties:
 281       - opendaylight-infra-properties:
 282           build-days-to-keep: '{build-days-to-keep}'
 283
 284     parameters:
 285       - opendaylight-infra-parameters:
 286           os-cloud: '{os-cloud}'
 287           project: '$GERRIT_PROJECT'
 288           branch: '{branch}'
 289           refspec: '$GERRIT_REFSPEC'
 290           artifacts: '{archive-artifacts}'
 291       - integration-distribution-branch:
 292           branch: '{branch}'
 293       - distribution-karaf-version:
 294           karaf-version: '{karaf-version}'
 295       - integration-distribution-git-url:
 296           git-url: '{git-url}'
 297       - maven-exec:
 298           maven-version: mvn35
 299
 300     scm:
 301       - integration-gerrit-scm:
 302           basedir: '$GERRIT_PROJECT'
 303           refspec: '$GERRIT_REFSPEC'
 304           branch: '{branch}'
 305       - integration-distribution-scm:
 306           branch: '{branch}'
 307
 308     wrappers:
 309       - opendaylight-infra-wrappers:
 310           build-timeout: '720'
 311
 312     triggers:
 313       - gerrit:
 314           server-name: '{gerrit-server-name}'
 315           trigger-on:
 316             - comment-added-contains-event:
 317                 comment-contains-value: 'test-integration'
 318           projects:
 319             - project-compare-type: 'ANT'
 320               project-pattern: '**'
 321               branches:
 322                 - branch-compare-type: 'ANT'
 323                   branch-pattern: '**/{branch}'
 324           skip-vote:
 325             successful: true
 326             failed: true
 327             unstable: true
 328             notbuilt: true
 329
 330     builders:
 331       - lf-infra-pre-build
 332       - integration-rebase-gerrit-patch
 333       - wipe-org-opendaylight-repo
 334       - lf-provide-maven-settings:
 335           global-settings-file: 'global-settings'
 336           settings-file: 'integration-settings'
 337       - lf-update-java-alternatives:
 338           java-version: '{java-version}'
 339       - maven-target:
 340           maven-version: 'mvn35'
 341           pom: '$GERRIT_PROJECT/pom.xml'
 342           goals: |
 343               clean install dependency:tree -DoutputFile=dependency_tree.txt
 344               -Pq
 345               -Dstream={stream}
 346               {opendaylight-infra-mvn-opts}
 347           # {opendaylight-infra-parallel-mvn-opts} is bad when there are multiple big features in SFT.
 348           java-opts:
 349             - '-Xmx4096m -Dmaven.compile.fork=true'
 350           settings: integration-settings
 351           settings-type: cfp
 352           global-settings: global-settings
 353           global-settings-type: cfp
 354       - maven-target:
 355           maven-version: mvn35
 356           pom: distribution/pom.xml
 357           goals: |
 358               clean install dependency:tree -DoutputFile=dependency_tree.txt
 359               {opendaylight-infra-mvn-opts}
 360           # {opendaylight-infra-parallel-mvn-opts} is bad when there are multiple big features in SFT.
 361           java-opts:
 362             - '-Xmx1024m -Dmaven.compile.fork=true'
 363           settings: integration-settings
 364           settings-type: cfp
 365           global-settings: global-settings
 366           global-settings-type: cfp
 367       - integration-upload-distribution:
 368           dist-pom: distribution/pom.xml
 369       - trigger-builds:
 370           - project: '{csit-list}'
 371             block: true
 372             predefined-parameters: |
 373               BUNDLE_URL=$BUNDLE_URL
 374               KARAF_VERSION=$KARAF_VERSION
 375
 376     publishers:
 377       - lf-infra-publish
 378       - email-notification:
 379           email-recipients: '{email-recipients}'
 380           email-prefix: '[$GERRIT_PROJECT]'
 381
 382
 383 - job-template:
 384     name: '{prefix}integration-multipatch-test-{stream}'
 385     # Goal: Build one or more patches and run all available system test on a distribution containing the change.
 386     # Operation: This job template checks out and builds in order each patch provided in job parameters, finally
 387     # it creates a distribution containing all patches and triggers the distribution test job which includes most
 388     # (if not all) available CSIT jobs.
 389
 390     id: inttest-multipatch
 391     project-type: freestyle
 392     node: centos7-builder-8c-8g
 393     concurrent: true
 394
 395     properties:
 396       - opendaylight-infra-properties:
 397           build-days-to-keep: '{build-days-to-keep}'
 398
 399     parameters:
 400       - string:
 401           name: PATCHES_TO_BUILD
 402           default: '$PATCHES_TO_BUILD'
 403           description: |
 404               Ordered list of patches to build in the format of project=changeset (checkout) or
 405               project:changeset (cherry-pick) for example: genius=32/53632/9,netvirt:59/50259/47.
 406       - string:
 407           name: BUILD_ORDER
 408           default: '{build-order}'
 409           description: 'Build order for multipatch test when topic is used'
 410       - string:
 411           name: BUILD_FAST
 412           default: 'true'
 413           description: 'Set true to build fast -Pq'
 414       - string:
 415           name: BUILD_NORMAL
 416           default: ''
 417           description: 'List of projects to build normal (no -Pq) when BUILD_FAST=true'
 418       - string:
 419           name: RUN_TEST
 420           default: 'false'
 421           description: 'Set true to run test after build'
 422       - integration-distribution-branch:
 423           branch: '{branch}'
 424       - integration-distribution-stream:
 425           stream: '{stream}'
 426       - distribution-karaf-version:
 427           karaf-version: '{karaf-version}'
 428       - opendaylight-infra-parameters:
 429           os-cloud: '{os-cloud}'
 430           project: 'integration/distribution'
 431           branch: '{branch}'
 432           refspec: 'refs/heads/{branch}'
 433           artifacts: '{archive-artifacts}'
 434       - lf-infra-maven-parameters:
 435           mvn-opts: ''
 436           mvn-params: ''
 437           mvn-version: mvn35
 438           staging-profile-id: ''  # Staging repos do not make sense for this job
 439
 440     wrappers:
 441       - opendaylight-infra-wrappers:
 442           build-timeout: '1200'
 443
 444     triggers:
 445       - gerrit:
 446           server-name: '{gerrit-server-name}'
 447           trigger-on:
 448             - comment-added-contains-event:
 449                 comment-contains-value: 'multipatch-build'
 450           projects:
 451             - project-compare-type: ANT
 452               project-pattern: '**'
 453               branches:
 454                 - branch-compare-type: ANT
 455                   branch-pattern: '**/{branch}'
 456
 457     builders:
 458       - lf-infra-pre-build
 459       - wipe-org-opendaylight-repo
 460       - lf-maven-install:
 461           mvn-version: '{mvn-version}'
 462       - lf-provide-maven-settings:
 463           global-settings-file: 'global-settings'
 464           settings-file: 'integration-distribution-settings'
 465       - lf-update-java-alternatives:
 466           java-version: '{java-version}'
 467       - integration-multipatch-builder
 468       - integration-upload-distribution:
 469           dist-pom: patch_tester/distribution/pom.xml
 470       - conditional-step:
 471           condition-kind: boolean-expression
 472           condition-expression: $RUN_TEST
 473           steps:
 474             - trigger-builds:
 475                 - project: '{csit-list}'
 476                   block: true
 477                   predefined-parameters: |
 478                     BUNDLE_URL=$BUNDLE_URL
 479                     KARAF_VERSION=$KARAF_VERSION
 480
 481     publishers:
 482       - lf-infra-publish
 483       - email-notification:
 484           email-recipients: '{email-recipients}'
 485           email-prefix: '[int/dist]'
 486
 487 - job-template:
 488     name: 'integration-distribution-weekly-test-trigger-{stream}'
 489
 490     project-type: freestyle
 491     node: queue-intque-2c-1g
 492
 493     properties:
 494       - opendaylight-infra-properties:
 495           build-days-to-keep: '{build-days-to-keep}'
 496
 497     parameters:
 498       - integration-bundle-url:
 499           bundle-url: '{bundle-url}'
 500       - distribution-karaf-version:
 501           karaf-version: '{karaf-version}'
 502
 503     triggers:
 504       - timed: '{schedule-weekly}'
 505
 506     builders:
 507       - trigger-builds:
 508           - project: '{csit-weekly-list}'
 509             block: true
 510             predefined-parameters: |
 511               BUNDLE_URL=$BUNDLE_URL
 512               KARAF_VERSION=$KARAF_VERSION
 513
 514     publishers:
 515       - email-notification:
 516           email-recipients: '{email-recipients}'
 517           email-prefix: '[integration]'
 518
 519 - job-template:
 520     name: 'integration-update-csit-tests-{stream}'
 521
 522     # This job's purpose is to update csit
 523     # tests to run in integration-distribution-test-{stream}
 524     # by updating integration files: csit-list-{stream}.inc
 525     # These files can be used as !include in int-dist-test-{stream}
 526
 527     ######################
 528     # Default parameters #
 529     ######################
 530
 531     git-url: '$GIT_URL/$PROJECT'
 532     submodule-disable: false
 533     submodule-recursive: true
 534     submodule-timeout: 10
 535
 536     #####################
 537     # Job Configuration #
 538     #####################
 539
 540     project-type: freestyle
 541     node: centos7-builder-2c-2g
 542
 543     properties:
 544       - lf-infra-properties:
 545           build-days-to-keep: 7
 546
 547     parameters:
 548       - lf-infra-parameters:
 549           project: releng/builder  # Hardcode as this job needs to pull releng/builder
 550           stream: '{stream}'
 551           branch: master  # Hardcode as releng/builder only has master
 552           lftools-version: '{lftools-version}'
 553       - string:
 554           name: CSIT_BLACKLIST
 555           default: '{csit-blacklist}'
 556           description: Space separated list of jobs strings to blacklist.
 557
 558     scm:
 559       - lf-infra-gerrit-scm:
 560           jenkins-ssh-credential: '{jenkins-ssh-credential}'
 561           git-url: '{git-url}'
 562           refspec: '$GERRIT_REFSPEC'
 563           branch: '$GERRIT_BRANCH'
 564           submodule-disable: '{submodule-disable}'
 565           submodule-recursive: '{submodule-recursive}'
 566           submodule-timeout: '{submodule-timeout}'
 567           choosing-strategy: default
 568
 569     wrappers:
 570       - lf-infra-wrappers:
 571           build-timeout: 10
 572           jenkins-ssh-credential: '{jenkins-ssh-credential}'
 573
 574     triggers:
 575       - reverse:
 576           jobs: builder-jjb-merge
 577
 578     builders:
 579       - lf-infra-pre-build
 580       - shell: !include-raw-escape: update-csit-tests.sh
 581       - shell: |
 582           git status
 583           git add jjb/integration/*.lst
 584       - lf-infra-push-gerrit-patch:
 585           project: 'releng/builder'
 586           gerrit-user: 'jenkins-$SILO'
 587           gerrit-host: 'git.opendaylight.org'
 588           gerrit-topic: 'auto-update-csit-jobs-{stream}'
 589           gerrit-commit-message: 'Auto Update CSIT Jobs to run for {stream}'
 590           reviewers-email: 'jluhrsen@gmail.com'
 591
 592     publishers:
 593       - lf-infra-publish
 594
 595 - project:
 596     name: integration-view
 597     views:
 598       - project-view
 599     project-name: integration
 600
 601 - view:
 602     name: csit-stable-ne
 603     description: 'List of stable integration jobs for managed projects'
 604     job-name:
 605       - aaa-csit-1node-authn-all-neon
 606       - bgpcep-csit-1node-throughpcep-all-neon
 607       - controller-csit-1node-akka1-all-neon
 608       - controller-csit-1node-benchmark-all-neon
 609       - controller-csit-1node-notifications-longevity-only-neon
 610       - controller-csit-1node-rest-cars-perf-all-neon
 611       - controller-csit-3node-benchmark-all-neon
 612       - controller-csit-3node-drb-precedence-longevity-only-neon
 613       - controller-csit-3node-rest-clust-cars-perf-ask-only-neon
 614       - controller-csit-3node-rest-clust-cars-perf-tell-only-neon
 615       - daexim-csit-1node-basic-only-neon
 616       - daexim-csit-3node-clustering-basic-only-neon
 617       - distribution-csit-1node-userfeatures-all-neon
 618       - genius-csit-1node-upstream-only-neon
 619       - genius-csit-3node-upstream-only-neon
 620       - lispflowmapping-csit-1node-msmr-all-neon
 621       - netconf-csit-1node-callhome-only-neon
 622       - netconf-csit-1node-userfeatures-all-neon
 623       - netvirt-csit-1node-0cmb-1ctl-2cmp-openstack-queens-upstream-stateful-neon
 624       - openflowplugin-csit-1node-flow-services-all-neon
 625       - openflowplugin-csit-1node-longevity-only-neon
 626       - openflowplugin-csit-1node-perf-bulkomatic-only-neon
 627       - openflowplugin-csit-1node-perf-stats-collection-only-neon
 628       - openflowplugin-csit-1node-sanity-only-neon
 629       - openflowplugin-csit-1node-scale-link-only-neon
 630       - openflowplugin-csit-1node-scale-only-neon
 631       - openflowplugin-csit-1node-scale-switch-only-neon
 632       - openflowplugin-csit-3node-clustering-bulkomatic-only-neon
 633       - openflowplugin-csit-3node-clustering-only-neon
 634       - ovsdb-csit-3node-upstream-clustering-only-neon
 635     view-type: list
 636     filter-executors: false
 637     filter-queue: false
 638     columns:
 639       - status
 640       - weather
 641       - job
 642       - last-success
 643       - last-failure
 644       - last-duration
 645       - build-button
 646       - jacoco
 647       - find-bugs
 648       - robot-list
 649     recurse: false