X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Fintegration%2Fintegration-test-jobs.yaml;h=9317ee400eabe6f0444012fe552dbf54307f19ea;hb=a2212d42de3e9efbeeae72d883a63069deed90ac;hp=3ebdc7d55a36b42b7adfaa944c6d2382adec756c;hpb=2d3990043987678c89e67a77b3d17029ac16cd35;p=releng%2Fbuilder.git

diff --git a/jjb/integration/integration-test-jobs.yaml b/jjb/integration/integration-test-jobs.yaml
index 3ebdc7d55..9317ee400 100644
--- a/jjb/integration/integration-test-jobs.yaml
+++ b/jjb/integration/integration-test-jobs.yaml
@@ -5,172 +5,49 @@
     project-name: integration-test
     test-branch: master
     jobs:
-      - '{project-name}-verify-tox-master'
-      - 'integration-csit-verify-1node-library'
       - 'integration-distribution-test-{stream}'
       - 'integration-sanity-test-{stream}'
       - 'integration-patch-test-{stream}'
       - 'integration-multipatch-test-{stream}'
       - 'integration-distribution-weekly-test-trigger-{stream}'
       - 'integration-distribution-high-frequency-test-trigger-{stream}'
+      - 'integration-update-csit-tests-{stream}'
+      - gerrit-tox-verify:
+          branch: master
+          stream: master
 
     # CSIT Lists in releng-defaults.yaml
     stream:
-      - nitrogen:
+      - oxygen:
           branch: 'master'
-          karaf-version: karaf3
-          jre: 'openjdk8'
-          csit-list: '{csit-list-nitrogen}'
+          csit-blacklist: longevity gate sanity capwap centinel coe didm iotdm jsonrpc lacp netide
+          csit-list: !include: csit-jobs-oxygen.lst
+          csit-sanity-list: '{csit-sanity-list-oxygen}'
+          csit-weekly-list: '{csit-weekly-list-oxygen}'
+          csit-high-frequency-list: '{csit-high-frequency-list-oxygen}'
+          mvn-version: 'mvn33'
+
+      - nitrogen:
+          branch: 'stable/nitrogen'
+          csit-blacklist: longevity gate sanity capwap centinel coe didm iotdm jsonrpc lacp netide tsdr
+          csit-list: !include: csit-jobs-nitrogen.lst
           csit-sanity-list: '{csit-sanity-list-nitrogen}'
           csit-weekly-list: '{csit-weekly-list-nitrogen}'
-          schedule-weekly: 'H 12 * * 0'
-          csit-high-frequency-list: ''
-          schedule-high-frequency: ''
+          csit-high-frequency-list: '{csit-high-frequency-list-nitrogen}'
+          mvn-version: 'mvn33'
 
       - carbon:
           branch: 'stable/carbon'
           karaf-version: karaf3
-          jdk: 'openjdk8'
-          jre: 'openjdk8'
-          csit-list: '{csit-list-carbon}'
+          csit-blacklist: longevity gate sanity capwap centinel coe jsonrpc
+          csit-list: !include: csit-jobs-carbon.lst
           csit-sanity-list: '{csit-sanity-list-carbon}'
           csit-weekly-list: '{csit-weekly-list-carbon}'
-          schedule-weekly: 'H 12 * * 0'
           csit-high-frequency-list: '{csit-high-frequency-list-carbon}'
-          schedule-high-frequency: 'H H/4 * * *'
-
-      - boron:
-          branch: 'stable/boron'
-          karaf-version: karaf3
-          jdk: 'openjdk8'
-          jre: 'openjdk8'
-          csit-list: '{csit-list-boron}'
-          csit-sanity-list: '{csit-sanity-list-boron}'
-          csit-weekly-list: '{csit-weekly-list-boron}'
-          schedule-weekly: 'H 12 * * 0'
-          csit-high-frequency-list: ''
-          schedule-high-frequency: ''
-
-      - beryllium:
-          branch: 'stable/beryllium'
-          karaf-version: karaf3
-          jdk: 'openjdk7'
-          jre: 'openjdk7'
-          csit-list: '{csit-list-beryllium}'
-          csit-sanity-list: ''
-          csit-weekly-list: ''
-          schedule-weekly: 'H 12 * * 0'
-          csit-high-frequency-list: ''
-          schedule-high-frequency: ''
+          mvn-version: 'mvn33'
 
     # tools system image
-    tools_system_image: Ubuntu 14.04 - mininet-ovs-25 - 20170210-0300
-
-# Template: integration-csit-verify-1node-library
-# Goal: Verify changes in csit folder not covered by projects suite verify jobs (e.g. libraries or variables)
-# Operation: This job template performs a base openflow test when there is a change in the libraries folder
-
-- job-template:
-    name: 'integration-csit-verify-1node-library'
-
-    project-type: freestyle
-    node: centos7-robot-2c-2g
-
-    # {stream} is already defined as a composite object, it cannot be overriden.
-    dist-stream: boron
-    # This is not a template, so define branch, scope and jre for distribution to test with.
-    branch: stable/boron
-    scope: only
-    jre: openjdk8
-
-    properties:
-      - opendaylight-infra-properties:
-          build-days-to-keep: '{build-days-to-keep}'
-
-    parameters:
-      - opendaylight-infra-parameters:
-          os-cloud: '{os-cloud}'
-          project: '{project}'
-          branch: '{test-branch}'
-          refspec: 'refs/heads/{test-branch}'
-          artifacts: '{archive-artifacts}'
-      - integration-distribution-branch:
-          branch: '{verify-branch}'
-      - integration-distribution-stream:
-          stream: '{verify-stream}'
-      - integration-bundle-url:
-          bundle-url: '{bundle-url}'
-      - integration-controller-scope:
-          controller-scope: '{scope}'
-      - integration-controller-features:
-          controller-features: >
-              odl-clustering-test-app,
-              odl-openflowplugin-flow-services-ui,
-              odl-openflowplugin-app-table-miss-enforcer,
-              odl-bgpcep-pcep,
-              odl-bgpcep-bgp,
-              odl-netconf-connector-all
-      # when upgrading to carbon you should replace odl-netconf-connector-all with odl-netconf-topology
-      # and change USE_NETCONF_CONNECTOR to False in test-options
-      - integration-test-options:
-          test-options: '-v TIMEOUT_BUG_4220:10s -v USE_NETCONF_CONNECTOR:True'
-      - integration-stream-test-plan:
-          stream-test-plan: 'test-libraries-{verify-stream}.txt'
-      - integration-test-plan:
-          test-plan: 'test-libraries.txt'
-      - integration-patch-refspec:
-          branch: '$GERRIT_REFSPEC'
-      - integration-jdk-version:
-          jdkversion: '{jre}'
-      - distribution-karaf-version:
-          karaf-version: '{verify-karaf-version}'
-
-    scm:
-      - integration-gerrit-scm:
-          basedir: 'test'
-          refspec: '$PATCHREFSPEC'
-          branch: '{test-branch}'
-
-    wrappers:
-      - opendaylight-infra-wrappers:
-          build-timeout: '{build-timeout}'
-
-    triggers:
-      - integration-trigger-patch-submitted:
-          server: '{server-name}'
-          project: '{project}'
-          branch: '{test-branch}'
-          files: 'csit/libraries/*'
-
-    builders:
-      - integration-infra-stack:
-          stack-template: '{stack-template}'
-          vm_0_count: '1'
-          vm_0_flavor: '{odl_system_flavor}'
-          vm_0_image: '{odl_system_image}'
-          vm_1_count: '{tools_system_count}'
-          vm_1_flavor: '{tools_system_flavor}'
-          vm_1_image: '{tools_system_image}'
-      - integration-install-robotframework
-      - inject:
-          properties-file: 'env.properties'
-      - integration-get-slave-addresses
-      - inject:
-          properties-file: 'slave_addresses.txt'
-      - integration-detect-variables
-      - integration-deploy-controller-run-test
-      - integration-cleanup-tmp
-
-    publishers:
-      - integration-robot:
-          unstable-if: 0.0
-          pass-if: 100.0
-      - email-notification:
-          email-recipients: '{email-recipients}'
-          email-prefix: '[integration]'
-      - integration-csit-archive-build
-      - opendaylight-infra-shiplogs:
-          maven-version: 'mvn33'
+    tools_system_image: ZZCI - Ubuntu 16.04 - mininet-ovs-25 - 20171208-1847
 
 # Template: integration-distribution-test-{stream}
 # Goal: Verify a distribution through all system test available
@@ -180,7 +57,7 @@
     name: 'integration-distribution-test-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
@@ -189,18 +66,23 @@
     parameters:
       - integration-bundle-url:
           bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     builders:
       - trigger-builds:
           - project: '{csit-list}'
             block: true
-            predefined-parameters:
+            predefined-parameters: |
               BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
+      - opendaylight-infra-generate-csit-status-report
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[integration]'
+      - lf-infra-publish
 
 # Template: integration-sanity-test-{stream}
 # Goal: Verify a distribution through sanity test
@@ -210,7 +92,7 @@
     name: 'integration-sanity-test-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
@@ -219,13 +101,16 @@
     parameters:
       - integration-bundle-url:
           bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     builders:
       - trigger-builds:
           - project: '{csit-sanity-list}'
             block: true
-            predefined-parameters:
+            predefined-parameters: |
               BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
       - email-notification:
@@ -241,7 +126,7 @@
     name: 'integration-patch-test-{stream}'
 
     project-type: freestyle
-    node: centos7-java-builder-2c-8g
+    node: centos7-builder-2c-8g
 
     properties:
       - opendaylight-infra-properties:
@@ -253,7 +138,7 @@
           project: '$GERRIT_PROJECT'
           branch: '{branch}'
           refspec: '$GERRIT_REFSPEC'
-          artifacts: '{archive-artifacts} **/dependency_tree.txt'
+          artifacts: '{archive-artifacts}'
       - integration-distribution-branch:
           branch: '{branch}'
       - distribution-karaf-version:
@@ -277,7 +162,7 @@
 
     triggers:
       - gerrit:
-          server-name: '{server-name}'
+          server-name: '{gerrit-server-name}'
           trigger-on:
             - comment-added-contains-event:
                 comment-contains-value: 'test-integration'
@@ -297,7 +182,7 @@
       - integration-rebase-gerrit-patch
       - wipe-org-opendaylight-repo
       - provide-maven-settings:
-          global-settings-file: 'odl-global-settings'
+          global-settings-file: 'global-settings'
           settings-file: 'integration-settings'
       - maven-target:
           maven-version: 'mvn33'
@@ -307,23 +192,24 @@
               -Pq
               -Dstream={stream}
               {opendaylight-infra-mvn-opts}
+          # TODO: {opendaylight-infra-parallel-mvn-opts}?
           java-opts:
             - '-Xmx4096m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
           settings: integration-settings
           settings-type: cfp
-          global-settings: odl-global-settings
+          global-settings: global-settings
           global-settings-type: cfp
       - maven-target:
           maven-version: mvn33
           pom: distribution/pom.xml
           goals: |
               clean install dependency:tree -DoutputFile=dependency_tree.txt
-              {opendaylight-infra-mvn-opts}
+              {opendaylight-infra-parallel-mvn-opts}
           java-opts:
             - '-Xmx1024m -XX:MaxPermSize=256m -Dmaven.compile.fork=true'
           settings: integration-settings
           settings-type: cfp
-          global-settings: odl-global-settings
+          global-settings: global-settings
           global-settings-type: cfp
       - integration-upload-distribution:
           dist-pom: distribution/pom.xml
@@ -335,8 +221,7 @@
               KARAF_VERSION=$KARAF_VERSION
 
     publishers:
-      - opendaylight-infra-shiplogs:
-          maven-version: 'mvn33'
+      - lf-infra-publish
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[$GERRIT_PROJECT]'
@@ -345,30 +230,12 @@
 - job-template:
     name: 'integration-multipatch-test-{stream}'
     # Goal: Build one or more patches and run all available system test on a distribution containing the change.
-    # Operation: This job template checks out and builds each patch provided in job parameters, creates a distribution
-    # containing these patches, and triggers the distribution test job which includes most (if not all) available
-    # CSIT jobs
-    # TODO: Move the following to appropriate readthedocs document.
-    # Recommended Sandbox usage:
-    # 0. Create this job and a CSIT job you want to run on the new .zip in Sandbox (disable timed triggers).
-    # 1. Copy the csit job name to clipboard.
-    # 2. Navigate to Sandbox page of this job.
-    # 3. Click "Configure".
-    # 4. Locate "Trigger/call builds on other projects" (under "Post Steps" tab).
-    # 5. Delete the long value of "Projects to build" field and paste the job name from clipboard (ctrl+a, ctrl+v).
-    # 6. Click "Save" button.
-    # 7. Click "Build with Parameters".
-    # yamllint disable-line rule:line-length
-    # 8. Start the job with your parameters, see https://wiki.opendaylight.org/view/Integration/Test/Running_System_Tests#Running_System_Tests_Using_Custom_Distribution_Built_From_Multiple_Patches
-    # 9. The multipatch job waits for the csit job to finish, so abort csit if you see something went wrong.
-    # 10. Navigate to Sandbox page of the csit job, click the run triggered from multipatch.
-    # 11. Click Parameters and copy the long value of BUNDLE_URL field to clipboard.
-    # 12. Run the same csit job woth different patchset or test options, or other csit jobs, with this BUNDLE_URL value
-    #     so you do not have to wait for multipatch build. Repeat step 12.
-    # 13. Currently, the .zip lasts 24 hours (?) so you may need to jump to step 2 (or 0) after some time.
+    # Operation: This job template checks out and builds in order each patch provided in job parameters, finally
+    # it creates a distribution containing all patches and triggers the distribution test job which includes most
+    # (if not all) available CSIT jobs.
 
     project-type: freestyle
-    node: centos7-java-builder-8c-8g
+    node: centos7-builder-4c-4g
     jdk: '{jdk}'
 
     properties:
@@ -382,6 +249,8 @@
           run-test: 'false'
       - integration-distribution-branch:
           branch: '{branch}'
+      - integration-distribution-stream:
+          stream: '{stream}'
       - distribution-karaf-version:
           karaf-version: '{karaf-version}'
       - opendaylight-infra-parameters:
@@ -389,7 +258,7 @@
           project: 'integration/distribution'
           branch: '{branch}'
           refspec: 'refs/heads/{branch}'
-          artifacts: '{archive-artifacts} **/dependency_tree.txt'
+          artifacts: '{archive-artifacts}'
       - maven-exec:
           maven-version: mvn33
 
@@ -397,27 +266,27 @@
       - opendaylight-infra-wrappers:
           build-timeout: '1200'
 
+    triggers:
+      - gerrit:
+          server-name: '{gerrit-server-name}'
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: 'multipatch-build:'
+          projects:
+            - project-compare-type: ANT
+              project-pattern: '**'
+              branches:
+                - branch-compare-type: ANT
+                  branch-pattern: '**/{branch}'
+
     builders:
       - wipe-org-opendaylight-repo
-      - integration-multipatch-builder
+      - lf-maven-install:
+          mvn-version: '{mvn-version}'
       - provide-maven-settings:
-          global-settings-file: 'odl-global-settings'
-          settings-file: 'yangtools-settings'
-      - maven-target:
-          maven-version: mvn33
-          pom: patch_tester/pom.xml
-          # TODO: Make Maven options configurable. Currently tests are not skipped, everything else is.
-          goals: |
-              clean install dependency:tree -DoutputFile=dependency_tree.txt
-              -Pq
-              -Dstream={stream}
-              {opendaylight-infra-mvn-opts}
-          java-opts:
-            - '-Xmx7168m -XX:MaxPermSize=1024m -Dmaven.compile.fork=true'
-          settings: integration-settings
-          settings-type: cfp
-          global-settings: odl-global-settings
-          global-settings-type: cfp
+          global-settings-file: 'global-settings'
+          settings-file: 'integration-distribution-settings'
+      - integration-multipatch-builder
       - integration-upload-distribution:
           dist-pom: patch_tester/distribution/pom.xml
       - conditional-step:
@@ -432,8 +301,7 @@
                     KARAF_VERSION=$KARAF_VERSION
 
     publishers:
-      - opendaylight-infra-shiplogs:
-          maven-version: 'mvn33'
+      - lf-infra-publish
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[int/dist]'
@@ -442,7 +310,7 @@
     name: 'integration-distribution-weekly-test-trigger-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
@@ -451,6 +319,8 @@
     parameters:
       - integration-bundle-url:
           bundle-url: '{bundle-url}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     triggers:
       - timed: '{schedule-weekly}'
@@ -459,8 +329,9 @@
       - trigger-builds:
           - project: '{csit-weekly-list}'
             block: true
-            predefined-parameters:
+            predefined-parameters: |
               BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
       - email-notification:
@@ -471,7 +342,7 @@
     name: 'integration-distribution-high-frequency-test-trigger-{stream}'
 
     project-type: freestyle
-    node: centos7-robot-2c-2g
+    node: centos7-robot-2c-8g
 
     properties:
       - opendaylight-infra-properties:
@@ -480,8 +351,8 @@
     parameters:
       - integration-bundle-url:
           bundle-url: '{bundle-url}'
-      - integration-jdk-version:
-          jdkversion: '{jre}'
+      - distribution-karaf-version:
+          karaf-version: '{karaf-version}'
 
     triggers:
       - timed: '{schedule-high-frequency}'
@@ -490,10 +361,79 @@
       - trigger-builds:
           - project: '{csit-high-frequency-list}'
             block: true
-            predefined-parameters:
+            predefined-parameters: |
               BUNDLE_URL=$BUNDLE_URL
+              KARAF_VERSION=$KARAF_VERSION
 
     publishers:
       - email-notification:
           email-recipients: '{email-recipients}'
           email-prefix: '[integration]'
+
+- job-template:
+    name: 'integration-update-csit-tests-{stream}'
+
+    # This job's purpose is to update csit
+    # tests to run in integration-distribution-test-{stream}
+    # by updating integration files: csit-list-{stream}.inc
+    # These files can be used as !include in int-dist-test-{stream}
+
+    ######################
+    # Default parameters #
+    ######################
+
+    git-url: '$GIT_URL/$PROJECT'
+    submodule-recursive: true
+
+    #####################
+    # Job Configuration #
+    #####################
+
+    project-type: freestyle
+    node: centos7-builder-2c-8g
+
+    properties:
+      - lf-infra-properties:
+          build-days-to-keep: 7
+
+    parameters:
+      - lf-infra-parameters:
+          project: releng/builder  # Hardcode as this job needs to pull releng/builder
+          stream: '{stream}'
+          branch: master  # Hardcode as releng/builder only has master
+          lftools-version: '{lftools-version}'
+      - string:
+          name: CSIT_BLACKLIST
+          default: '{csit-blacklist}'
+          description: Space separated list of jobs strings to blacklist.
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: '{submodule-recursive}'
+          choosing-strategy: default
+
+    wrappers:
+      - lf-infra-wrappers:
+          build-timeout: 10
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    triggers:
+      - reverse:
+          jobs: builder-jjb-merge
+
+    builders:
+      - shell: !include-raw-escape: update-csit-tests.sh
+      - shell: |
+          git status
+          git add jjb/integration/*.lst
+      - opendaylight-infra-push-gerrit-patch:
+          project: "releng/builder"
+          gerrit-topic: "auto-update-csit-jobs-{stream}"
+          gerrit-commit-message: "Auto Update CSIT Jobs to run for {stream}"
+
+    publishers:
+      - lf-infra-publish