Merge "Increase ocata packer timeout from 60m to 75m"
[releng/builder.git] / jjb / integration / integration-test-jobs.yaml
index ebdc8e320a4bc8744211d755ba9fe3bdf73a8def..9317ee400eabe6f0444012fe552dbf54307f19ea 100644 (file)
     project-name: integration-test
     test-branch: master
     jobs:
-      - '{project-name}-verify-tox-master'
-      - 'integration-csit-verify-1node-library'
       - 'integration-distribution-test-{stream}'
       - 'integration-sanity-test-{stream}'
       - 'integration-patch-test-{stream}'
       - 'integration-multipatch-test-{stream}'
       - 'integration-distribution-weekly-test-trigger-{stream}'
+      - 'integration-distribution-high-frequency-test-trigger-{stream}'
+      - 'integration-update-csit-tests-{stream}'
+      - gerrit-tox-verify:
+          branch: master
+          stream: master
 
     # CSIT Lists in releng-defaults.yaml
     stream:
-      - nitrogen:
+      - oxygen:
           branch: 'master'
-          jre: 'openjdk8'
-          csit-list: '{csit-list-nitrogen}'
+          csit-blacklist: longevity gate sanity capwap centinel coe didm iotdm jsonrpc lacp netide
+          csit-list: !include: csit-jobs-oxygen.lst
+          csit-sanity-list: '{csit-sanity-list-oxygen}'
+          csit-weekly-list: '{csit-weekly-list-oxygen}'
+          csit-high-frequency-list: '{csit-high-frequency-list-oxygen}'
+          mvn-version: 'mvn33'
+
+      - nitrogen:
+          branch: 'stable/nitrogen'
+          csit-blacklist: longevity gate sanity capwap centinel coe didm iotdm jsonrpc lacp netide tsdr
+          csit-list: !include: csit-jobs-nitrogen.lst
           csit-sanity-list: '{csit-sanity-list-nitrogen}'
           csit-weekly-list: '{csit-weekly-list-nitrogen}'
-          schedule-weekly: 'H 12 * * 0'
+          csit-high-frequency-list: '{csit-high-frequency-list-nitrogen}'
+          mvn-version: 'mvn33'
+
       - carbon:
           branch: 'stable/carbon'
-          jdk: 'openjdk8'
-          jre: 'openjdk8'
-          csit-list: '{csit-list-carbon}'
+          karaf-version: karaf3
+          csit-blacklist: longevity gate sanity capwap centinel coe jsonrpc
+          csit-list: !include: csit-jobs-carbon.lst
           csit-sanity-list: '{csit-sanity-list-carbon}'
           csit-weekly-list: '{csit-weekly-list-carbon}'
-          schedule-weekly: 'H 12 * * 0'
-
-      - boron:
-          branch: 'stable/boron'
-          jdk: 'openjdk8'
-          jre: 'openjdk8'
-          csit-list: '{csit-list-boron}'
-          csit-sanity-list: '{csit-sanity-list-boron}'
-          csit-weekly-list: '{csit-weekly-list-boron}'
-          schedule-weekly: 'H 12 * * 0'
-
-      - beryllium:
-          branch: 'stable/beryllium'
-          jdk: 'openjdk7'
-          jre: 'openjdk7'
-          csit-list: '{csit-list-beryllium}'
-          csit-sanity-list: ''
-          csit-weekly-list: ''
-          schedule-weekly: 'H 12 * * 0'
+          csit-high-frequency-list: '{csit-high-frequency-list-carbon}'
+          mvn-version: 'mvn33'
 
     # tools system image
-    tools_system_image: Ubuntu 14.04 - mininet-ovs-25 - 20170210-0300
-
-# Template: integration-csit-verify-1node-library
-# Goal: Verify changes in csit folder not covered by projects suite verify jobs (e.g. libraries or variables)
-# Operation: This job template performs a base openflow test when there is a change in the libraries folder
-
-- job-template:
-    name: 'integration-csit-verify-1node-library'
-
-    project-type: freestyle
-    node: centos7-robot-2c-2g
-
-    # {stream} is already defined as a composite object, it cannot be overriden.
-    dist-stream: boron
-    # This is not a template, so define branch, scope and jre for distribution to test with.
-    branch: stable/boron
-    scope: only
-    jre: openjdk8
-
-    properties:
-      - opendaylight-infra-properties:
-          build-days-to-keep: '{build-days-to-keep}'
-
-    parameters:
-      - opendaylight-infra-parameters:
-          os-cloud: '{os-cloud}'
-          project: '{project}'
-          branch: '{test-branch}'
-          refspec: 'refs/heads/{test-branch}'
-          artifacts: '{archive-artifacts}'
-      - integration-distribution-branch:
-          branch: '{branch}'
-      - integration-distribution-stream:
-          stream: '{dist-stream}'
-      - integration-bundleurl:
-          bundleurl: '{bundleurl}'
-      - integration-controller-scope:
-          controller-scope: '{scope}'
-      - integration-controller-features:
-          controller-features: >
-              odl-clustering-test-app,
-              odl-openflowplugin-flow-services-ui,
-              odl-openflowplugin-app-table-miss-enforcer,
-              odl-bgpcep-pcep,
-              odl-bgpcep-bgp,
-              odl-netconf-connector-all
-      # when upgrading to carbon you should replace odl-netconf-connector-all with odl-netconf-topology
-      # and change USE_NETCONF_CONNECTOR to False in test-options
-      - integration-test-options:
-          test-options: '-v TIMEOUT_BUG_4220:10s -v USE_NETCONF_CONNECTOR:True'
-      - integration-stream-test-plan:
-          stream-test-plan: 'test-libraries-{dist-stream}.txt'
-      - integration-test-plan:
-          test-plan: 'test-libraries.txt'
-      - integration-patch-refspec:
-          branch: '$GERRIT_REFSPEC'
-      - integration-jdk-version:
-          jdkversion: '{jre}'
-
-    scm:
-      - integration-gerrit-scm:
-          basedir: 'test'
-          refspec: '$PATCHREFSPEC'
-          branch: '{test-branch}'
-
-    wrappers:
-      - opendaylight-infra-wrappers:
-          build-timeout: '{build-timeout}'
-
-    triggers:
-      - integration-trigger-patch-submitted:
-          server: '{server-name}'
-          project: '{project}'
-          branch: '{test-branch}'
-          files: 'csit/libraries/*'
-
-    builders:
-      - integration-infra-stack:
-          stack-template: '{stack-template}'
-          vm_0_count: '1'
-          vm_0_flavor: '{odl_system_flavor}'
-          vm_0_image: '{odl_system_image}'
-          vm_1_count: '{tools_system_count}'
-          vm_1_flavor: '{tools_system_flavor}'
-          vm_1_image: '{tools_system_image}'
-      - integration-install-robotframework
-      - inject:
-          properties-file: 'env.properties'
-      - integration-get-slave-addresses
-      - inject:
-          properties-file: 'slave_addresses.txt'
-      - integration-get-bundle-vars
-      - inject:
-          properties-file: 'bundle_vars.txt'
-      - integration-deploy-controller-run-test
-      - integration-cleanup-tmp
-
-    publishers:
-      - integration-robot:
-          unstable-if: 0.0
-          pass-if: 100.0
-      - email-notification:
-          email-recipients: '{email-recipients}'
-          email-prefix: '[integration]'
-      - integration-csit-archive-build
-      - opendaylight-infra-shiplogs:
-          maven-version: 'mvn33'
+    tools_system_image: ZZCI - Ubuntu 16.04 - mininet-ovs-25 - 20171208-1847
 
 # Template: integration-distribution-test-{stream}
 # Goal: Verify a distribution through all system test available
     name: 'integration-distribution-test-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
           build-days-to-keep: '{build-days-to-keep}'
 
     parameters:
-      - integration-bundleurl:
-          bundleurl: '{bundleurl}'
-      - integration-jdk-version:
-          jdkversion: '{jre}'
+      - integration-bundle-url:
+          bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     builders:
       - trigger-builds:
           - project: '{csit-list}'
             block: true
-            predefined-parameters:
-              BUNDLEURL=$BUNDLEURL
+            predefined-parameters: |
+              BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
+      - opendaylight-infra-generate-csit-status-report
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[integration]'
+      - lf-infra-publish
 
 # Template: integration-sanity-test-{stream}
 # Goal: Verify a distribution through sanity test
     name: 'integration-sanity-test-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
           build-days-to-keep: '{build-days-to-keep}'
 
     parameters:
-      - integration-bundleurl:
-          bundleurl: '{bundleurl}'
-      - integration-jdk-version:
-          jdkversion: '{jre}'
+      - integration-bundle-url:
+          bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     builders:
       - trigger-builds:
           - project: '{csit-sanity-list}'
             block: true
-            predefined-parameters:
-              BUNDLEURL=$BUNDLEURL
+            predefined-parameters: |
+              BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
       - email-notification:
 - job-template:
     name: 'integration-patch-test-{stream}'
 
-    project-type: maven
-    node: centos7-java-builder-2c-8g
-    jdk: '{jdk}'
+    project-type: freestyle
+    node: centos7-builder-2c-8g
 
     properties:
       - opendaylight-infra-properties:
           project: '$GERRIT_PROJECT'
           branch: '{branch}'
           refspec: '$GERRIT_REFSPEC'
-          artifacts: '{archive-artifacts} **/dependency_tree.txt'
+          artifacts: '{archive-artifacts}'
       - integration-distribution-branch:
           branch: '{branch}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
       - integration-distribution-git-url:
           git-url: '{git-url}'
+      - maven-exec:
+          maven-version: mvn33
 
     scm:
       - integration-gerrit-scm:
 
     triggers:
       - gerrit:
-          server-name: '{server-name}'
+          server-name: '{gerrit-server-name}'
           trigger-on:
             - comment-added-contains-event:
                 comment-contains-value: 'test-integration'
             unstable: true
             notbuilt: true
 
-    prebuilders:
+    builders:
       - integration-rebase-gerrit-patch
-      - integration-get-bundle-url
-      - inject:
-          properties-file: 'bundle.txt'
       - wipe-org-opendaylight-repo
       - provide-maven-settings:
-          global-settings-file: 'odl-global-settings'
+          global-settings-file: 'global-settings'
           settings-file: 'integration-settings'
       - maven-target:
           maven-version: 'mvn33'
           pom: '$GERRIT_PROJECT/pom.xml'
-          goals: >
+          goals: |
               clean install dependency:tree -DoutputFile=dependency_tree.txt
-              -V -B -Pq -Djenkins
-              -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
+              -Pq
               -Dstream={stream}
+              {opendaylight-infra-mvn-opts}
+          # TODO: {opendaylight-infra-parallel-mvn-opts}?
           java-opts:
             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
-          settings: 'integration-settings'
+          settings: integration-settings
           settings-type: cfp
-          global-settings: 'odl-global-settings'
+          global-settings: global-settings
           global-settings-type: cfp
-
-    maven:
-      maven-name: 'mvn33'
-      root-pom: 'distribution/pom.xml'
-      goals: >
-          clean install dependency:tree -DoutputFile=dependency_tree.txt
-          -V -B -Djenkins
-          -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
-      maven-opts: '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
-      settings: 'integration-settings'
-      settings-type: cfp
-      global-settings: 'odl-global-settings'
-      global-settings-type: cfp
-      ignore-upstream-changes: true
-      post-step-run-condition: 'SUCCESS'
-
-    postbuilders:
+      - maven-target:
+          maven-version: mvn33
+          pom: distribution/pom.xml
+          goals: |
+              clean install dependency:tree -DoutputFile=dependency_tree.txt
+              {opendaylight-infra-parallel-mvn-opts}
+          java-opts:
+            - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
+          settings: integration-settings
+          settings-type: cfp
+          global-settings: global-settings
+          global-settings-type: cfp
+      - integration-upload-distribution:
+          dist-pom: distribution/pom.xml
       - trigger-builds:
           - project: '{csit-list}'
             block: true
-            predefined-parameters:
-              BUNDLEURL=$BUNDLEURL
+            predefined-parameters: |
+              BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
-      - opendaylight-infra-shiplogs:
-          maven-version: 'mvn33'
+      - lf-infra-publish
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[$GERRIT_PROJECT]'
 
-# Template: integration-multipatch-test-{stream}
-# Goal: Build one or more patches and run all available system test on a distribution containing the change
-# Operation: This job template checks out and builds each patch provided in job parameters, creates a distribution
-# containing these patches, and triggers the distribution test job which includes most (if not all) available
-# CSIT jobs
 
 - job-template:
     name: 'integration-multipatch-test-{stream}'
+    # Goal: Build one or more patches and run all available system test on a distribution containing the change.
+    # Operation: This job template checks out and builds in order each patch provided in job parameters, finally
+    # it creates a distribution containing all patches and triggers the distribution test job which includes most
+    # (if not all) available CSIT jobs.
 
-    project-type: maven
-    node: centos7-java-builder-8c-8g
+    project-type: freestyle
+    node: centos7-builder-4c-4g
     jdk: '{jdk}'
 
     properties:
           run-test: 'false'
       - integration-distribution-branch:
           branch: '{branch}'
+      - integration-distribution-stream:
+          stream: '{stream}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
       - opendaylight-infra-parameters:
           os-cloud: '{os-cloud}'
           project: 'integration/distribution'
           branch: '{branch}'
           refspec: 'refs/heads/{branch}'
-          artifacts: '{archive-artifacts} **/dependency_tree.txt'
-
-    prebuilders:
-      - wipe-org-opendaylight-repo
-      - integration-multipatch-builder
-      - inject:
-          properties-file: 'bundle.txt'
-      - provide-maven-settings:
-          global-settings-file: 'odl-global-settings'
-          settings-file: 'yangtools-settings'
+          artifacts: '{archive-artifacts}'
+      - maven-exec:
+          maven-version: mvn33
 
     wrappers:
       - opendaylight-infra-wrappers:
           build-timeout: '1200'
 
-    maven:
-      maven-name: 'mvn33'
-      root-pom: 'patch_tester/pom.xml'
-      # TODO: Make Maven options configurable. Currently tests are not skipped, everything else is.
-      goals: >
-          clean install dependency:tree -DoutputFile=dependency_tree.txt
-          -f patch_tester/pom.xml -V -B -Pq -Djenkins
-          -Dmaven.repo.local=/tmp/r -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r
-          -Dstream={stream}
-      maven-opts: '-Xmx7168m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
-      settings: 'yangtools-settings'
-      settings-type: cfp
-      global-settings: 'odl-global-settings'
-      global-settings-type: cfp
-      post-step-run-condition: 'SUCCESS'
-
-    postbuilders:
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: 'multipatch-build:'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '**'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+
+    builders:
+      - wipe-org-opendaylight-repo
+      - lf-maven-install:
+          mvn-version: '{mvn-version}'
+      - provide-maven-settings:
+          global-settings-file: 'global-settings'
+          settings-file: 'integration-distribution-settings'
+      - integration-multipatch-builder
+      - integration-upload-distribution:
+          dist-pom: patch_tester/distribution/pom.xml
       - conditional-step:
           condition-kind: boolean-expression
           condition-expression: $RUNTEST
             - trigger-builds:
                 - project: '{csit-list}'
                   block: true
-                  predefined-parameters:
-                    BUNDLEURL=$BUNDLEURL
+                  predefined-parameters: |
+                    BUNDLE_URL=$BUNDLE_URL
+                    KARAF_VERSION=$KARAF_VERSION
 
     publishers:
-      - opendaylight-infra-shiplogs:
-          maven-version: 'mvn33'
+      - lf-infra-publish
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[int/dist]'
     name: 'integration-distribution-weekly-test-trigger-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
           build-days-to-keep: '{build-days-to-keep}'
 
     parameters:
-      - integration-bundleurl:
-          bundleurl: '{bundleurl}'
-      - integration-jdk-version:
-          jdkversion: '{jre}'
+      - integration-bundle-url:
+          bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     triggers:
       - timed: '{schedule-weekly}'
       - trigger-builds:
           - project: '{csit-weekly-list}'
             block: true
-            predefined-parameters:
-              BUNDLEURL=$BUNDLEURL
+            predefined-parameters: |
+              BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[integration]'
+
+- job-template:
+    name: 'integration-distribution-high-frequency-test-trigger-{stream}'
+
+    project-type: freestyle
+    node: centos7-robot-2c-8g
+
+    properties:
+      - opendaylight-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+
+    parameters:
+      - integration-bundle-url:
+          bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
+
+    triggers:
+      - timed: '{schedule-high-frequency}'
+
+    builders:
+      - trigger-builds:
+          - project: '{csit-high-frequency-list}'
+            block: true
+            predefined-parameters: |
+              BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
+
+    publishers:
+      - email-notification:
+          email-recipients: '{email-recipients}'
+          email-prefix: '[integration]'
+
+- job-template:
+    name: 'integration-update-csit-tests-{stream}'
+
+    # This job's purpose is to update csit
+    # tests to run in integration-distribution-test-{stream}
+    # by updating integration files: csit-list-{stream}.inc
+    # These files can be used as !include in int-dist-test-{stream}
+
+    ######################
+    # Default parameters #
+    ######################
+
+    git-url: '$GIT_URL/$PROJECT'
+    submodule-recursive: true
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    project-type: freestyle
+    node: centos7-builder-2c-8g
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: 7
+
+    parameters:
+      - lf-infra-parameters:
+          project: releng/builder  # Hardcode as this job needs to pull releng/builder
+          stream: '{stream}'
+          branch: master  # Hardcode as releng/builder only has master
+          lftools-version: '{lftools-version}'
+      - string:
+          name: CSIT_BLACKLIST
+          default: '{csit-blacklist}'
+          description: Space separated list of jobs strings to blacklist.
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: 10
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    triggers:
+      - reverse:
+          jobs: builder-jjb-merge
+
+    builders:
+      - shell: !include-raw-escape: update-csit-tests.sh
+      - shell: |
+          git status
+          git add jjb/integration/*.lst
+      - opendaylight-infra-push-gerrit-patch:
+          project: "releng/builder"
+          gerrit-topic: "auto-update-csit-jobs-{stream}"
+          gerrit-commit-message: "Auto Update CSIT Jobs to run for {stream}"
+
+    publishers:
+      - lf-infra-publish