Add integration-replace-block-text macro

Part of refactor of distribution jobs
workflow. This auto updates
csit run list for integration-test-jobs.yaml

Moved storage of tests to run from integration-test-jobs
to variables in releng-defaults.
Variables are named: csit-list-carbon,
csit-list-boron, csit-list-beryllium

Change-Id: Ib989ce0cc1b206596c8b3cc4f069e5c714309f7f
Signed-off-by: Marcus G K Williams <marcus.williams@intel.com>

diff --git a/jjb/integration/include-raw-integration-replace-block-text.sh b/jjb/integration/include-raw-integration-replace-block-text.sh
new file mode 100644 (file)
index 0000000..a21f7e2
--- /dev/null
+++ b/jjb/integration/include-raw-integration-replace-block-text.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+starting_regex={starting-regex}
+ending_regex={ending-regex}
+file_with_changes_to_insert={file-with-changes-to-insert}
+file_to_change={file-to-change}
+output=$(sed -e "/$starting_regex/,/$ending_regex/{{ /$starting_regex/{{p; r ${{file_with_changes_to_insert}}
+        }}; /$ending_regex/p; d }}"  ${{file_to_change}})
+echo "$output" > ${{file_to_change}}

diff --git a/jjb/integration/integration-macros.yaml b/jjb/integration/integration-macros.yaml
index 83c2eb375d9e20d66d9d65fcf937888f6bd5dc3b..d92bfd5698b993673447719c93f119e0d89228cf 100644 (file)
--- a/jjb/integration/integration-macros.yaml
+++ b/jjb/integration/integration-macros.yaml
@@ -427,6 +427,25 @@
             !include-raw:
                 - include-raw-integration-list-jobs.sh
 
+# Macro: integration-replace-block-text
+# Operation: using starting and ending regex to
+# replace a block of text in a file with the text in
+# another file. Regex should be basic and work with sed
+# Used by: builder-merge job template
+#
+# This script requires the following JJB variables to be passed in:
+#   {starting-regex}
+#   {ending-regex}
+#   {file-with-changes-to-insert}
+#   {file-to-change}
+
+- builder:
+    name: integration-replace-block-text
+    builders:
+        - shell:
+            !include-raw:
+                - include-raw-integration-replace-block-text.sh
+
 ##############
 # Publishers #
 ##############

diff --git a/jjb/integration/integration-test-jobs.yaml b/jjb/integration/integration-test-jobs.yaml
index 052be2f5c28bb29d79dabe8ef05f9472e6f7d25f..99156640334159527accd34f3f22de265a97ba54 100644 (file)
--- a/jjb/integration/integration-test-jobs.yaml
+++ b/jjb/integration/integration-test-jobs.yaml
@@ -10,168 +10,25 @@
     project: integration/test
     project-name: integration-test
 
+    # CSIT Lists in releng-defaults.yaml
     stream:
         - carbon:
             branch: 'master'
             jdk: 'openjdk8'
             jre: 'openjdk8'
-            csit-list: >
-                controller-csit-3node-clustering-only-{stream},
-                controller-csit-3node-clustering-all-{stream},
-                integration-distribution-deploy-{stream},
-                integration-distribution-offline-{stream},
-                l2switch-csit-1node-switch-only-{stream},
-                l2switch-csit-1node-switch-all-{stream},
-                l2switch-csit-1node-scalability-only-{stream},
-                l2switch-csit-1node-scalability-all-{stream},
-                lispflowmapping-csit-1node-msmr-only-{stream},
-                lispflowmapping-csit-1node-msmr-all-{stream},
-                nemo-csit-1node-engine-all-{stream},
-                nemo-csit-1node-engine-only-{stream},
-                netvirt-csit-1node-openstack-mitaka-upstream-learn-{stream},
-                netvirt-csit-1node-openstack-mitaka-upstream-transparent-{stream},
-                netvirt-legacy-csit-1node-openstack-basic-all-{stream},
-                netvirt-legacy-csit-1node-openstack-basic-only-{stream},
-                netvirt-legacy-csit-1node-openstack-mitaka-upstream-{stream},
-                netvirt-legacy-csit-3node-clustering-only-{stream},
-                ocpplugin-csit-1node-get-only-{stream},
-                ocpplugin-csit-1node-get-all-{stream},
-                ocpplugin-csit-1node-scalability-only-{stream},
-                ocpplugin-csit-1node-scalability-all-{stream},
-                openflowplugin-csit-1node-flow-services-only-{stream},
-                openflowplugin-csit-1node-flow-services-all-{stream},
-                openflowplugin-csit-1node-scalability-only-{stream},
-                openflowplugin-csit-1node-cbench-performance-only-{stream},
-                openflowplugin-csit-1node-config-performance-only-{stream},
-                openflowplugin-csit-3node-clustering-only-{stream},
-                ovsdb-csit-1node-southbound-all-{stream},
-                ovsdb-csit-1node-southbound-only-{stream},
-                ovsdb-csit-3node-clustering-only-{stream},
-                packetcable-csit-1node-pcmm-only-{stream},
-                packetcable-csit-1node-pcmm-all-{stream},
-                sxp-csit-1node-basic-only-{stream},
-                sxp-csit-1node-filtering-all-{stream},
-                sxp-csit-1node-topology-only-{stream},
-                sxp-csit-1node-topology-all-{stream},
-                topoprocessing-csit-1node-topology-operations-all-{stream},
-                topoprocessing-csit-1node-topology-operations-only-{stream},
-                tsdr-csit-1node-cassandra-datastore-only-{stream},
-                tsdr-csit-1node-hbase-datastore-all-{stream},
-                tsdr-csit-1node-hbase-datastore-only-{stream},
-                tsdr-csit-1node-hsqldb-datastore-all-{stream},
-                tsdr-csit-1node-hsqldb-datastore-only-{stream},
-                usc-csit-1node-channel-only-{stream},
-                usc-csit-1node-channel-all-{stream},
-                usc-csit-1node-tcp-only-{stream},
-                usc-csit-1node-tcp-all-{stream},
-                vtn-csit-1node-manager-only-{stream},
-                vtn-csit-1node-manager-all-{stream},
-                vtn-csit-1node-coordinator-only-{stream}
+            csit-list: {csit-list-carbon}
 
         - boron:
             branch: 'stable/boron'
             jdk: 'openjdk8'
             jre: 'openjdk8'
-            csit-list: >
-                controller-csit-3node-clustering-only-{stream},
-                controller-csit-3node-clustering-all-{stream},
-                integration-distribution-deploy-{stream},
-                integration-distribution-offline-{stream},
-                l2switch-csit-1node-switch-only-{stream},
-                l2switch-csit-1node-switch-all-{stream},
-                l2switch-csit-1node-scalability-only-{stream},
-                l2switch-csit-1node-scalability-all-{stream},
-                lispflowmapping-csit-1node-msmr-only-{stream},
-                lispflowmapping-csit-1node-msmr-all-{stream},
-                nemo-csit-1node-engine-all-{stream},
-                nemo-csit-1node-engine-only-{stream},
-                netvirt-csit-1node-openstack-mitaka-upstream-learn-{stream},
-                netvirt-csit-1node-openstack-mitaka-upstream-transparent-{stream},
-                netvirt-legacy-csit-1node-openstack-basic-all-{stream},
-                netvirt-legacy-csit-1node-openstack-basic-only-{stream},
-                netvirt-legacy-csit-1node-openstack-mitaka-upstream-{stream},
-                netvirt-legacy-csit-3node-clustering-only-{stream},
-                ocpplugin-csit-1node-get-only-{stream},
-                ocpplugin-csit-1node-get-all-{stream},
-                ocpplugin-csit-1node-scalability-only-{stream},
-                ocpplugin-csit-1node-scalability-all-{stream},
-                openflowplugin-csit-1node-flow-services-only-{stream},
-                openflowplugin-csit-1node-flow-services-all-{stream},
-                openflowplugin-csit-1node-scalability-only-{stream},
-                openflowplugin-csit-1node-cbench-performance-only-{stream},
-                openflowplugin-csit-1node-config-performance-only-{stream},
-                openflowplugin-csit-3node-clustering-only-{stream},
-                ovsdb-csit-1node-southbound-all-{stream},
-                ovsdb-csit-1node-southbound-only-{stream},
-                ovsdb-csit-3node-clustering-only-{stream},
-                packetcable-csit-1node-pcmm-only-{stream},
-                packetcable-csit-1node-pcmm-all-{stream},
-                sxp-csit-1node-basic-only-{stream},
-                sxp-csit-1node-filtering-all-{stream},
-                sxp-csit-1node-topology-only-{stream},
-                sxp-csit-1node-topology-all-{stream},
-                topoprocessing-csit-1node-topology-operations-all-{stream},
-                topoprocessing-csit-1node-topology-operations-only-{stream},
-                tsdr-csit-1node-cassandra-datastore-only-{stream},
-                tsdr-csit-1node-hbase-datastore-only-{stream},
-                tsdr-csit-1node-hsqldb-datastore-only-{stream},
-                usc-csit-1node-channel-only-{stream},
-                usc-csit-1node-channel-all-{stream},
-                usc-csit-1node-tcp-only-{stream},
-                usc-csit-1node-tcp-all-{stream},
-                vtn-csit-1node-manager-only-{stream},
-                vtn-csit-1node-manager-all-{stream},
-                vtn-csit-1node-coordinator-only-{stream}
+            csit-list: {csit-list-boron}
 
         - beryllium:
             branch: 'stable/beryllium'
             jdk: 'openjdk7'
             jre: 'openjdk7'
-            csit-list: >
-                aaa-csit-1node-authn-only-{stream},
-                aaa-csit-1node-authn-all-{stream},
-                bgpcep-csit-1node-userfeatures-only-{stream},
-                bgpcep-csit-1node-userfeatures-all-{stream},
-                capwap-csit-1node-ac-only-{stream},
-                capwap-csit-1node-ac-all-{stream},
-                integration-distribution-offline-{stream},
-                integration-distribution-deploy-{stream},
-                l2switch-csit-1node-switch-only-{stream},
-                l2switch-csit-1node-scalability-only-{stream},
-                l2switch-csit-1node-scalability-all-{stream},
-                lispflowmapping-csit-1node-msmr-only-{stream},
-                lispflowmapping-csit-1node-msmr-all-{stream},
-                nemo-csit-1node-engine-all-{stream},
-                nemo-csit-1node-engine-only-{stream},
-                nic-csit-1node-basic-all-{stream},
-                nic-csit-1node-basic-only-{stream},
-                openflowplugin-csit-1node-flow-services-only-{stream},
-                openflowplugin-csit-1node-flow-services-all-{stream},
-                openflowplugin-csit-1node-scalability-only-{stream},
-                openflowplugin-csit-1node-cbench-performance-only-{stream},
-                openflowplugin-csit-1node-config-performance-only-{stream},
-                openflowplugin-csit-3node-clustering-only-{stream},
-                ovsdb-csit-1node-southbound-all-{stream},
-                ovsdb-csit-1node-southbound-only-{stream},
-                packetcable-csit-1node-pcmm-only-{stream},
-                packetcable-csit-1node-pcmm-all-{stream},
-                snmp-csit-1node-basic-only-{stream},
-                snmp-csit-1node-basic-all-{stream},
-                sxp-csit-1node-basic-only-{stream},
-                sxp-csit-1node-filtering-only-{stream},
-                sxp-csit-1node-filtering-all-{stream},
-                topoprocessing-csit-1node-topology-operations-all-{stream},
-                topoprocessing-csit-1node-topology-operations-only-{stream},
-                tsdr-csit-1node-cassandra-datastore-only-{stream},
-                tsdr-csit-1node-hbase-datastore-only-{stream},
-                tsdr-csit-1node-hsqldb-datastore-only-{stream},
-                usc-csit-1node-channel-only-{stream},
-                usc-csit-1node-channel-all-{stream},
-                usc-csit-1node-tcp-only-{stream},
-                usc-csit-1node-tcp-all-{stream},
-                vtn-csit-1node-manager-only-{stream},
-                vtn-csit-1node-manager-all-{stream},
-                vtn-csit-1node-coordinator-only-{stream}
+            csit-list: {csit-list-beryllium}
 
     # tools system image
     tools_system_image: Ubuntu 14.04 - mininet-ovs-2.5 - 20170117-0004

diff --git a/jjb/releng-defaults.yaml b/jjb/releng-defaults.yaml
index 4168020566d1ed04ec8d163bf33d558112b9c3f4..cb245e9bca1ad86645cede91c4734e0e9fd1ccdb 100644 (file)
--- a/jjb/releng-defaults.yaml
+++ b/jjb/releng-defaults.yaml
@@ -109,3 +109,153 @@
     10-plot-group: ''
     10-inclusion-flag: 'off'
     10-exclude-value: ''
+
+    # CSIT Test Lists
+    csit-list-carbon: >
+        controller-csit-3node-clustering-only-carbon,
+        controller-csit-3node-clustering-all-carbon,
+        integration-distribution-deploy-carbon,
+        integration-distribution-offline-carbon,
+        l2switch-csit-1node-switch-only-carbon,
+        l2switch-csit-1node-switch-all-carbon,
+        l2switch-csit-1node-scalability-only-carbon,
+        l2switch-csit-1node-scalability-all-carbon,
+        lispflowmapping-csit-1node-msmr-only-carbon,
+        lispflowmapping-csit-1node-msmr-all-carbon,
+        nemo-csit-1node-engine-all-carbon,
+        nemo-csit-1node-engine-only-carbon,
+        netvirt-csit-1node-openstack-mitaka-upstream-learn-carbon,
+        netvirt-csit-1node-openstack-mitaka-upstream-transparent-carbon,
+        netvirt-legacy-csit-1node-openstack-basic-all-carbon,
+        netvirt-legacy-csit-1node-openstack-basic-only-carbon,
+        netvirt-legacy-csit-1node-openstack-mitaka-upstream-carbon,
+        netvirt-legacy-csit-3node-clustering-only-carbon,
+        ocpplugin-csit-1node-get-only-carbon,
+        ocpplugin-csit-1node-get-all-carbon,
+        ocpplugin-csit-1node-scalability-only-carbon,
+        ocpplugin-csit-1node-scalability-all-carbon,
+        openflowplugin-csit-1node-flow-services-only-carbon,
+        openflowplugin-csit-1node-flow-services-all-carbon,
+        openflowplugin-csit-1node-scalability-only-carbon,
+        openflowplugin-csit-1node-cbench-performance-only-carbon,
+        openflowplugin-csit-1node-config-performance-only-carbon,
+        openflowplugin-csit-3node-clustering-only-carbon,
+        ovsdb-csit-1node-southbound-all-carbon,
+        ovsdb-csit-1node-southbound-only-carbon,
+        ovsdb-csit-3node-clustering-only-carbon,
+        packetcable-csit-1node-pcmm-only-carbon,
+        packetcable-csit-1node-pcmm-all-carbon,
+        sxp-csit-1node-basic-only-carbon,
+        sxp-csit-1node-filtering-all-carbon,
+        sxp-csit-1node-topology-only-carbon,
+        sxp-csit-1node-topology-all-carbon,
+        topoprocessing-csit-1node-topology-operations-all-carbon,
+        topoprocessing-csit-1node-topology-operations-only-carbon,
+        tsdr-csit-1node-cassandra-datastore-only-carbon,
+        tsdr-csit-1node-hbase-datastore-all-carbon,
+        tsdr-csit-1node-hbase-datastore-only-carbon,
+        tsdr-csit-1node-hsqldb-datastore-all-carbon,
+        tsdr-csit-1node-hsqldb-datastore-only-carbon,
+        usc-csit-1node-channel-only-carbon,
+        usc-csit-1node-channel-all-carbon,
+        usc-csit-1node-tcp-only-carbon,
+        usc-csit-1node-tcp-all-carbon,
+        vtn-csit-1node-manager-only-carbon,
+        vtn-csit-1node-manager-all-carbon,
+        vtn-csit-1node-coordinator-only-carbon
+    csit-list-boron: >
+        controller-csit-3node-clustering-only-boron,
+        controller-csit-3node-clustering-all-boron,
+        integration-distribution-deploy-boron,
+        integration-distribution-offline-boron,
+        l2switch-csit-1node-switch-only-boron,
+        l2switch-csit-1node-switch-all-boron,
+        l2switch-csit-1node-scalability-only-boron,
+        l2switch-csit-1node-scalability-all-boron,
+        lispflowmapping-csit-1node-msmr-only-boron,
+        lispflowmapping-csit-1node-msmr-all-boron,
+        nemo-csit-1node-engine-all-boron,
+        nemo-csit-1node-engine-only-boron,
+        netvirt-csit-1node-openstack-mitaka-upstream-learn-boron,
+        netvirt-csit-1node-openstack-mitaka-upstream-transparent-boron,
+        netvirt-legacy-csit-1node-openstack-basic-all-boron,
+        netvirt-legacy-csit-1node-openstack-basic-only-boron,
+        netvirt-legacy-csit-1node-openstack-mitaka-upstream-boron,
+        netvirt-legacy-csit-3node-clustering-only-boron,
+        ocpplugin-csit-1node-get-only-boron,
+        ocpplugin-csit-1node-get-all-boron,
+        ocpplugin-csit-1node-scalability-only-boron,
+        ocpplugin-csit-1node-scalability-all-boron,
+        openflowplugin-csit-1node-flow-services-only-boron,
+        openflowplugin-csit-1node-flow-services-all-boron,
+        openflowplugin-csit-1node-scalability-only-boron,
+        openflowplugin-csit-1node-cbench-performance-only-boron,
+        openflowplugin-csit-1node-config-performance-only-boron,
+        openflowplugin-csit-3node-clustering-only-boron,
+        ovsdb-csit-1node-southbound-all-boron,
+        ovsdb-csit-1node-southbound-only-boron,
+        ovsdb-csit-3node-clustering-only-boron,
+        packetcable-csit-1node-pcmm-only-boron,
+        packetcable-csit-1node-pcmm-all-boron,
+        sxp-csit-1node-basic-only-boron,
+        sxp-csit-1node-filtering-all-boron,
+        sxp-csit-1node-topology-only-boron,
+        sxp-csit-1node-topology-all-boron,
+        topoprocessing-csit-1node-topology-operations-all-boron,
+        topoprocessing-csit-1node-topology-operations-only-boron,
+        tsdr-csit-1node-cassandra-datastore-only-boron,
+        tsdr-csit-1node-hbase-datastore-only-boron,
+        tsdr-csit-1node-hsqldb-datastore-only-boron,
+        usc-csit-1node-channel-only-boron,
+        usc-csit-1node-channel-all-boron,
+        usc-csit-1node-tcp-only-boron,
+        usc-csit-1node-tcp-all-boron,
+        vtn-csit-1node-manager-only-boron,
+        vtn-csit-1node-manager-all-boron,
+        vtn-csit-1node-coordinator-only-boron
+    csit-list-beryllium: >
+        aaa-csit-1node-authn-only-beryllium,
+        aaa-csit-1node-authn-all-beryllium,
+        bgpcep-csit-1node-userfeatures-only-beryllium,
+        bgpcep-csit-1node-userfeatures-all-beryllium,
+        capwap-csit-1node-ac-only-beryllium,
+        capwap-csit-1node-ac-all-beryllium,
+        integration-distribution-offline-beryllium,
+        integration-distribution-deploy-beryllium,
+        l2switch-csit-1node-switch-only-beryllium,
+        l2switch-csit-1node-scalability-only-beryllium,
+        l2switch-csit-1node-scalability-all-beryllium,
+        lispflowmapping-csit-1node-msmr-only-beryllium,
+        lispflowmapping-csit-1node-msmr-all-beryllium,
+        nemo-csit-1node-engine-all-beryllium,
+        nemo-csit-1node-engine-only-beryllium,
+        nic-csit-1node-basic-all-beryllium,
+        nic-csit-1node-basic-only-beryllium,
+        openflowplugin-csit-1node-flow-services-only-beryllium,
+        openflowplugin-csit-1node-flow-services-all-beryllium,
+        openflowplugin-csit-1node-scalability-only-beryllium,
+        openflowplugin-csit-1node-cbench-performance-only-beryllium,
+        openflowplugin-csit-1node-config-performance-only-beryllium,
+        openflowplugin-csit-3node-clustering-only-beryllium,
+        ovsdb-csit-1node-southbound-all-beryllium,
+        ovsdb-csit-1node-southbound-only-beryllium,
+        packetcable-csit-1node-pcmm-only-beryllium,
+        packetcable-csit-1node-pcmm-all-beryllium,
+        snmp-csit-1node-basic-only-beryllium,
+        snmp-csit-1node-basic-all-beryllium,
+        sxp-csit-1node-basic-only-beryllium,
+        sxp-csit-1node-filtering-only-beryllium,
+        sxp-csit-1node-filtering-all-beryllium,
+        topoprocessing-csit-1node-topology-operations-all-beryllium,
+        topoprocessing-csit-1node-topology-operations-only-beryllium,
+        tsdr-csit-1node-cassandra-datastore-only-beryllium,
+        tsdr-csit-1node-hbase-datastore-only-beryllium,
+        tsdr-csit-1node-hsqldb-datastore-only-beryllium,
+        usc-csit-1node-channel-only-beryllium,
+        usc-csit-1node-channel-all-beryllium,
+        usc-csit-1node-tcp-only-beryllium,
+        usc-csit-1node-tcp-all-beryllium,
+        vtn-csit-1node-manager-only-beryllium,
+        vtn-csit-1node-manager-all-beryllium,
+        vtn-csit-1node-coordinator-only-beryllium
+        # CSIT TESTS END SED MARKER

diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index 4c00833dbea5ef59038f2ec00d319f308a843042..4d892c74f99814c5109a82e1f15a5c4fcbd3cc36 100644 (file)
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -201,6 +201,47 @@
                       pattern: jjb/**
 
     builders:
+        - shell: |
+            git checkout -b auto-update-csit-jobs
+        - integration-list-jobs:
+            blacklist: "longevity gate"
+            search_string: "csit"
+            stream: "carbon"
+            jobs-filename: "csit_jobs_carbon.lst"
+        - integration-list-jobs:
+            blacklist: "longevity gate"
+            search_string: "csit"
+            stream: "boron"
+            jobs-filename: "csit_jobs_boron.lst"
+        - integration-list-jobs:
+            blacklist: "longevity gate"
+            search_string: "csit"
+            stream: "beryllium"
+            jobs-filename: "csit_jobs_beryllium.lst"
+        - integration-replace-block-text:
+            starting-regex: "'csit-list-carbon: >'"
+            ending-regex: "'csit-list-boron: >'"
+            file-with-changes-to-insert: "'csit_jobs_carbon.lst'"
+            file-to-change: "'jjb/releng-defaults.yaml'"
+        - integration-replace-block-text:
+            starting-regex: "'csit-list-boron: >'"
+            ending-regex: "'csit-list-beryllium: >'"
+            file-with-changes-to-insert: "'csit_jobs_boron.lst'"
+            file-to-change: "'jjb/releng-defaults.yaml'"
+        - integration-replace-block-text:
+            starting-regex: "'csit-list-beryllium: >'"
+            ending-regex: "'# CSIT TESTS END SED MARKER'"
+            file-with-changes-to-insert: "'csit_jobs_beryllium.lst'"
+            file-to-change: "'jjb/releng-defaults.yaml'"
+        - shell: |
+            git status
+            git add jjb/releng-defaults.yaml
+        - opendaylight-infra-push-gerrit-patch:
+            project: "releng/builder"
+            gerrit-topic: "auto-update-csit-jobs"
+            gerrit-commit-message: "Auto Update CSIT Jobs to run"
+        - shell: |
+            git checkout master
         - shell: |
             pip freeze
             jenkins-jobs --version