Allow usage of MRI distribution in csit 89/97189/8
authorTomas Cere <tomas.cere@pantheon.tech>
Mon, 2 Aug 2021 08:38:30 +0000 (10:38 +0200)
committerTomas Cere <tomas.cere@pantheon.tech>
Thu, 19 Aug 2021 14:09:25 +0000 (16:09 +0200)
Add new config values for karaf_version parameter
Add necessary handling of the MRI distros in the
integration scripts
Change netconf jobs to use the MRI distro

Change-Id: I036098e24eaffa979477242b9329c97997cfdd71
Signed-off-by: Tomas Cere <tomas.cere@pantheon.tech>
14 files changed:
jjb/integration/common-functions.sh
jjb/integration/distribution/distribution-check-bootup.sh
jjb/integration/integration-compare-distributions.sh
jjb/integration/integration-detect-variables.sh
jjb/integration/integration-macros.yaml
jjb/integration/integration-set-variables.sh
jjb/integration/integration-templates.yaml
jjb/integration/integration-upload-distribution.sh
jjb/netconf/netconf-clustering-scale.yaml
jjb/netconf/netconf-clustering.yaml
jjb/netconf/netconf-csit-callhome.yaml
jjb/netconf/netconf-patch-test.yaml
jjb/netconf/netconf-scale.yaml
jjb/netconf/netconf-userfeatures.yaml

index d921d0d0a1b65ed418e8e1efdc96112a30d49e35..cfd8e39da635c8b36adf41b6380c1e3f4f2b3a56 100644 (file)
@@ -856,7 +856,11 @@ function get_nodes_list() {
 
 function get_features() {
     if [ "${CONTROLLERSCOPE}" == 'all' ]; then
-        ACTUALFEATURES="odl-integration-compatible-with-all,${CONTROLLERFEATURES}"
+        if [ "$KARAF_PROJECT" == "integration" ]; then
+            ACTUALFEATURES="odl-integration-compatible-with-all,${CONTROLLERFEATURES}"
+        else
+            ACTUALFEATURES="odl-infrautils-ready,${CONTROLLERFEATURES}"
+        fi
         # if CONTROLLERMEM still is the default 2G and was not overridden by a
         # custom job, then we need to make sure to increase it because "all"
         # features can be heavy
@@ -916,9 +920,12 @@ if [[ "$KARAF_VERSION" == "karaf4" ]]; then
     FEATURE_TEST_STRING="features-test"
 fi
 
-sed -ie "s%\\(featuresRepositories=\\|featuresRepositories =\\)%featuresRepositories = mvn:org.opendaylight.integration/\${FEATURE_TEST_STRING}/${BUNDLE_VERSION}/xml/features,mvn:org.apache.karaf.decanter/apache-karaf-decanter/1.2.0/xml/features,%g" ${FEATURESCONF}
-if [[ ! -z "${REPO_URL}" ]]; then
-   sed -ie "s%featuresRepositories =%featuresRepositories = ${REPO_URL},%g" ${FEATURESCONF}
+# only manipulate feature repo in integration distro
+if [[ "$KARAF_PROJECT" == "integration" ]]; then
+       sed -ie "s%\\(featuresRepositories=\\|featuresRepositories =\\)%featuresRepositories = mvn:org.opendaylight.integration/\${FEATURE_TEST_STRING}/${BUNDLE_VERSION}/xml/features,mvn:org.apache.karaf.decanter/apache-karaf-decanter/1.2.0/xml/features,%g" ${FEATURESCONF}
+       if [[ ! -z "${REPO_URL}" ]]; then
+          sed -ie "s%featuresRepositories =%featuresRepositories = ${REPO_URL},%g" ${FEATURESCONF}
+       fi
 fi
 cat ${FEATURESCONF}
 
index 0cd69a969ff69afc6f09ebbcf49cfa6769ba919d..1dad0d1f1646de7db1e3fb54478ead1193042d6b 100644 (file)
@@ -1,8 +1,13 @@
 #!/bin/bash
-if [[ -n "${CONTROLLERFEATURES}" ]]; then
-    ACTUALFEATURES="odl-integration-all,${CONTROLLERFEATURES}"
+# do not add integration features in MRI projects
+if [[ "$KARAF_PROJECT" == integration ]]; then
+    if [[ -n "${CONTROLLERFEATURES}" ]]; then
+        ACTUALFEATURES="odl-integration-all,${CONTROLLERFEATURES}"
+    else
+        ACTUALFEATURES="odl-integration-all"
+    fi
 else
-    ACTUALFEATURES="odl-integration-all"
+    ACTUALFEATURES="${CONTROLLERFEATURES}"
 fi
 
 if [[ "${JOB_NAME}" == *"distribution-sanity"* ]]; then
@@ -35,19 +40,23 @@ if [[ "$KARAF_VERSION" == "karaf3" ]]; then
     FEATURE_TEST_STRING="features-integration-test"
 fi
 
-sed -ie "s%\(featuresRepositories= \|featuresRepositories = \)%featuresRepositories = mvn:org.opendaylight.integration/${FEATURE_TEST_STRING}//xml/features,%g" "${FEATURESCONF}"
+# only replace feature repo in integration/distro, MRI projects need to pull in
+# the features they need by themselves
+if [[ "$KARAF_PROJECT" == integration ]]; then
+    sed -ie "s%\(featuresRepositories= \|featuresRepositories = \)%featuresRepositories = mvn:org.opendaylight.integration/${FEATURE_TEST_STRING}//xml/features,%g" "${FEATURESCONF}"
 
-if [[ -n "${REPO_URL}" ]]; then
-   # sed below will fail if it finds space between feature repos.
-   REPO_URL_NO_SPACE="$(echo -e "${REPO_URL}" | tr -d '[:space:]')"
-   sed -ie "s%featuresRepositories = %featuresRepositories = ${REPO_URL_NO_SPACE},%g" "${FEATURESCONF}"
-fi
+    if [[ -n "${REPO_URL}" ]]; then
+       # sed below will fail if it finds space between feature repos.
+       REPO_URL_NO_SPACE="$(echo -e "${REPO_URL}" | tr -d '[:space:]')"
+       sed -ie "s%featuresRepositories = %featuresRepositories = ${REPO_URL_NO_SPACE},%g" "${FEATURESCONF}"
+    fi
 
-# Add actual boot features.
-# sed below will fail if it finds space between feature repos.
-FEATURES_NO_SPACE="$(echo -e "${ACTUALFEATURES}" | tr -d '[:space:]')"
-sed -ie "s/\(featuresBoot= \|featuresBoot = \)/featuresBoot = ${FEATURES_NO_SPACE},/g" "${FEATURESCONF}"
-cat "${FEATURESCONF}"
+    # Add actual boot features.
+    # sed below will fail if it finds space between feature repos.
+    FEATURES_NO_SPACE="$(echo -e "${ACTUALFEATURES}" | tr -d '[:space:]')"
+    sed -ie "s/\(featuresBoot= \|featuresBoot = \)/featuresBoot = ${FEATURES_NO_SPACE},/g" "${FEATURESCONF}"
+    cat "${FEATURESCONF}"
+fi
 
 echo "Configuring the log..."
 LOGCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.ops4j.pax.logging.cfg"
index f31f79e387f51ef495101a3933368d3dec36392f..040b6cfa3f92715c54e6253fc05e124c05fd4d5f 100644 (file)
@@ -13,6 +13,12 @@ DISTROBRANCH=${DISTROBRANCH:-$GERRIT_BRANCH}
 # Obtain current pom.xml of integration/distribution, correct branch.
 if [[ "$KARAF_ARTIFACT" == "opendaylight" ]]; then
     wget "http://${GERRIT_PATH}/gitweb?p=integration/distribution.git;a=blob_plain;f=opendaylight/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+elif [[ "$KARAF_ARTIFACT" == "karaf" ]]; then
+    wget "http://${GERRIT_PATH}/gitweb?p=integration/distribution.git;a=blob_plain;f=pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+elif [[ "$KARAF_ARTIFACT" == "netconf-karaf" ]]; then
+    wget "http://${GERRIT_PATH}/gitweb?p=${KARAF_PROJECT}.git;a=blob_plain;f=karaf/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+elif [[ "$KARAF_ARTIFACT" == "controller-test-karaf" ]]; then
+    wget "http://${GERRIT_PATH}/gitweb?p=${KARAF_PROJECT}.git;a=blob_plain;f=karaf/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
 else
     wget "http://${GERRIT_PATH}/gitweb?p=integration/distribution.git;a=blob_plain;f=pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
 fi
@@ -21,7 +27,7 @@ fi
 BUNDLE_VERSION=$(xpath pom.xml '/project/version/text()' 2> /dev/null)
 echo "Bundle version is ${BUNDLE_VERSION}"
 # Acquire the timestamp information from maven-metadata.xml
-NEXUSPATH="${NEXUSURL_PREFIX}/${ODL_NEXUS_REPO}/org/opendaylight/integration/${KARAF_ARTIFACT}"
+NEXUSPATH="${NEXUSURL_PREFIX}/${ODL_NEXUS_REPO}/org/opendaylight/${KARAF_PROJECT}/${KARAF_ARTIFACT}"
 wget "${NEXUSPATH}/${BUNDLE_VERSION}/maven-metadata.xml"
 
 # shellcheck disable=SC2181
index f2acb4c6e957acad7a48966daf7f0803ee0e12dd..fddb2f9670396a5b9bc1778e4fe2f2d1d563c829 100644 (file)
@@ -9,6 +9,12 @@ if [ "${BUNDLE_URL}" == 'last' ]; then
     # Obtain current pom.xml of integration/distribution, correct branch.
     if [[ "$KARAF_ARTIFACT" == "opendaylight" ]]; then
         wget "http://${GERRIT_PATH}/gitweb?p=integration/distribution.git;a=blob_plain;f=opendaylight/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+    elif [[ "$KARAF_ARTIFACT" == "karaf" ]]; then
+        wget "http://${GERRIT_PATH}/gitweb?p=integration/distribution.git;a=blob_plain;f=pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+    elif [[ "$KARAF_ARTIFACT" == "netconf-karaf" ]]; then
+        wget "http://${GERRIT_PATH}/gitweb?p=${KARAF_PROJECT}.git;a=blob_plain;f=karaf/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+    elif [[ "$KARAF_ARTIFACT" == "controller-test-karaf" ]]; then
+        wget "http://${GERRIT_PATH}/gitweb?p=${KARAF_PROJECT}.git;a=blob_plain;f=karaf/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
     else
         wget "http://${GERRIT_PATH}/gitweb?p=integration/distribution.git;a=blob_plain;f=pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
     fi
@@ -16,7 +22,7 @@ if [ "${BUNDLE_URL}" == 'last' ]; then
     BUNDLE_VERSION="$(xpath pom.xml '/project/version/text()' 2> /dev/null)"
     echo "Bundle version is ${BUNDLE_VERSION}"
     # Acquire the timestamp information from maven-metadata.xml
-    NEXUSPATH="${NEXUSURL_PREFIX}/${ODL_NEXUS_REPO}/org/opendaylight/integration/${KARAF_ARTIFACT}"
+    NEXUSPATH="${NEXUSURL_PREFIX}/${ODL_NEXUS_REPO}/org/opendaylight/${KARAF_PROJECT}/${KARAF_ARTIFACT}"
     wget "${NEXUSPATH}/${BUNDLE_VERSION}/maven-metadata.xml"
     less "maven-metadata.xml"
     TIMESTAMP="$(xpath maven-metadata.xml "//snapshotVersion[extension='zip'][1]/value/text()" 2>/dev/null)"
index 2075a8cd7cbd438ea11744cc600563c5185cf400..9522fd937f99ed347d892269fb4b8a3029623f95 100644 (file)
       - string:
           name: KARAF_VERSION
           default: "{karaf-version}"
-          description: "karaf3, karaf4, odl, affects ODL deploy scripts"
+          description: "karaf3, karaf4, odl, controller, netconf affects ODL deploy scripts"
 
 - parameter:
     name: integration-bundle-url
index 76e8d726321eb8487926a30f686edb567730a4c6..45bf097cc9ffa700e31ec777113c2b0868de2f0b 100644 (file)
@@ -2,10 +2,19 @@
 
 if [[ "$KARAF_VERSION" == "odl" ]]; then
     KARAF_ARTIFACT="opendaylight"
+    KARAF_PROJECT="integration"
 elif [[ "$KARAF_VERSION" == "karaf3" ]]; then
     KARAF_ARTIFACT="distribution-karaf"
+    KARAF_PROJECT="integration"
+elif [[ "$KARAF_VERSION" == "controller" ]]; then
+    KARAF_ARTIFACT="controller-test-karaf"
+    KARAF_PROJECT="controller"
+elif [[ "$KARAF_VERSION" == "netconf" ]]; then
+    KARAF_ARTIFACT="netconf-karaf"
+    KARAF_PROJECT="netconf"
 else
     KARAF_ARTIFACT="karaf"
+    KARAF_PROJECT="integration"
 fi
 
 if [ "$JDKVERSION" == 'openjdk11' ]; then
@@ -17,12 +26,14 @@ elif [ "$JDKVERSION" == 'openjdk8' ]; then
 fi
 
 echo "Karaf artifact is ${KARAF_ARTIFACT}"
+echo "Karaf project is ${KARAF_PROJECT}"
 echo "Java home is ${JAVA_HOME}"
 
 # The following is not a shell file, double quotes would be literal.
 cat > "${WORKSPACE}/set_variables.env" <<EOF
 JAVA_HOME=${JAVA_HOME}
 KARAF_ARTIFACT=${KARAF_ARTIFACT}
+KARAF_PROJECT=${KARAF_PROJECT}
 EOF
 
 # vim: ts=4 sw=4 sts=4 et ft=sh :
index 662bf7fa6a4c3f02f5d010f18563c2a895aae919..81e19b4a9188698fd3830f444c790129545c4deb 100644 (file)
             Install a custom OVS. It can be an OVS version to build (like
             v2.9.2 or v2.6.1-nsh) or a yum repo url to pull from.
 
+# Template: {project}-mri-patch-test-{stream}
+# Goal: Build a patch and run project specific system test on a mri projects distribution containing the change
+# Operation: This job template builds a patch, uses the projects distribution and triggers the project system
+# test defined in {csit-list} when gerrit receives test-{project} keyword
+
+- job-template:
+    name: "{prefix}{project}-mri-patch-test-{feature}-{stream}"
+    id: inttest-mri-patch-test
+    <<: *inttest_csit_common
+
+    build-timeout: 720
+
+    parameters:
+      - opendaylight-infra-parameters:
+          os-cloud: "{os-cloud}"
+          project: "{project}"
+          branch: "{branch}"
+          refspec: "refs/heads/{branch}"
+          artifacts: "{archive-artifacts}"
+      - integration-distribution-branch:
+          branch: "{branch}"
+      - distribution-karaf-version:
+          karaf-version: "{karaf-version}"
+      - integration-distribution-git-url:
+          git-url: "{git-url}"
+      - string:
+          name: feature
+          default: "all"
+          description: "Specific feature test for patch"
+      - maven-exec:
+          maven-version: mvn35
+
+    scm:
+      - integration-gerrit-scm:
+          basedir: "{project}"
+          refspec: "$GERRIT_REFSPEC"
+          branch: "{branch}"
+      - integration-distribution-scm:
+          branch: "{branch}"
+
+    triggers:
+      - gerrit:
+          server-name: "{gerrit-server-name}"
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: "test-{project}-all"
+            - comment-added-contains-event:
+                comment-contains-value: "test-{project}-{feature}"
+          projects:
+            - project-compare-type: "ANT"
+              project-pattern: "{project}"
+              branches:
+                - branch-compare-type: "ANT"
+                  branch-pattern: "**/{branch}"
+          skip-vote:
+            successful: true
+            failed: true
+            unstable: true
+            notbuilt: true
+
+    builders:
+      - integration-rebase-gerrit-patch
+      - wipe-org-opendaylight-repo
+      - lf-update-java-alternatives:
+          java-version: "{java-version}"
+      - maven-target:
+          maven-version: mvn35
+          pom: "{project}/pom.xml"
+          goals: |
+            clean install dependency:tree
+            -Pq
+            -Dgitid.skip=false
+            -Dmaven.gitcommitid.skip=false
+            -DgenerateReports=false
+            -Dstream={stream}
+            {opendaylight-infra-mvn-opts}
+          # {opendaylight-infra-parallel-mvn-opts} is bad when there are multiple big features in SFT.
+          java-opts:
+            - "-Xmx4096m -Dmaven.compile.fork=true"
+          settings: integration-settings
+          settings-type: cfp
+          global-settings: global-settings
+          global-settings-type: cfp
+      - integration-upload-distribution:
+          dist-pom: "{project}/pom.xml"
+      - trigger-builds:
+          - project: "{csit-list}"
+            block: true
+            predefined-parameters: |
+              BUNDLE_URL=$BUNDLE_URL
+              GERRIT_BRANCH=$GERRIT_BRANCH
+              GERRIT_PROJECT=$GERRIT_PROJECT
+              GERRIT_REFSPEC=$GERRIT_REFSPEC
+              KARAF_VERSION=$KARAF_VERSION
+
+    publishers:
+      - email-notification:
+          email-recipients: "{email-recipients}"
+          email-prefix: "[{project}]"
+
 # Template: {project}-patch-test-{stream}
 # Goal: Build a patch and run project specific system test on a distribution containing the change
 # Operation: This job template builds a patch, creates a distribution containing the patch, and
index 354401b1411dd3ab2ccf268322e2caf6a2afdb19..07bba5e61308dda7d9b44aa06dceea089cfbb3dd 100644 (file)
@@ -4,7 +4,18 @@ set -xeu -o pipefail
 BUNDLE_VERSION=$(xpath "${BUNDLE_POM}" '/project/version/text()' 2> /dev/null)
 BUNDLEFOLDER="${KARAF_ARTIFACT}-${BUNDLE_VERSION}"
 BUNDLE="${BUNDLEFOLDER}.zip"
-BUNDLE_PATH="/tmp/r/org/opendaylight/integration/${KARAF_ARTIFACT}/${BUNDLE_VERSION}/${BUNDLE}"
+BUNDLE_PATH="/tmp/r/org/opendaylight/${KARAF_PROJECT}/${KARAF_ARTIFACT}/${BUNDLE_VERSION}/${BUNDLE}"
+if [[ "$KARAF_PROJECT" == "integration" ]]; then
+    GROUP_ID="org.opendaylight.${KARAF_PROJECT}.${GERRIT_PROJECT//\//.}"
+else
+    GROUP_ID="org.opendaylight.${KARAF_PROJECT}"
+fi
+
+echo "Bundle folder is ${BUNDLEFOLDER}"
+echo "Bundle is ${BUNDLE}"
+echo "Bundle path is ${BUNDLE_PATH}"
+echo "Group ID is ${GROUP_ID}"
+
 ls -l "${BUNDLE_PATH}"
 LOG_FILE='integration-upload-distribution.log'
 echo "Uploading distribution to Nexus..."
@@ -15,7 +26,7 @@ echo "Uploading distribution to Nexus..."
     -Dfile="${BUNDLE_PATH}" \
     -DrepositoryId=opendaylight-snapshot \
     -Durl="$ODLNEXUSPROXY/content/repositories/opendaylight.snapshot/" \
-    -DgroupId="org.opendaylight.integration.${GERRIT_PROJECT//\//.}" \
+    -DgroupId="${GROUP_ID}" \
     -DartifactId="${KARAF_ARTIFACT}" \
     -Dversion="${BUNDLE_VERSION}" \
     -Dpackaging=zip \
index f73b589f3229efd74d54118cdfb82fd225258283..1fb4d5fafba96f28739e7517eaf69791076c1a3c 100644 (file)
@@ -22,6 +22,8 @@
           branch: "stable/aluminium"
           jre: "openjdk11"
 
+    karaf-version: netconf
+
     install:
       - only:
           scope: "only"
index 5efea41d877f23deccbb5724dbfaa2e7d107ba02..3e98cc930d0a0a5fad28551415ad653573953889 100644 (file)
@@ -22,6 +22,8 @@
           branch: "stable/aluminium"
           jre: "openjdk11"
 
+    karaf-version: netconf
+
     install:
       - only:
           scope: "only"
index da070bb208b9a1f8944f1e10d14f89bec6def784..06d26ba611185a1106082f2bcd972e7cd9588be9 100644 (file)
@@ -24,6 +24,8 @@
           branch: "stable/aluminium"
           jre: "openjdk11"
 
+    karaf-version: netconf
+
     install:
       - only:
           scope: "only"
index 141da5e1c9f46dd16a551ac53a7bbd545ebc5db3..4b6984a168a41e7ecb8766cf8da1bc4347234834 100644 (file)
@@ -2,7 +2,7 @@
 - project:
     name: netconf-patch-test
     jobs:
-      - inttest-patch-test
+      - inttest-mri-patch-test
 
     project: "netconf"
 
@@ -19,6 +19,8 @@
 
     os-branch: "queens"
 
+    karaf-version: netconf
+
     feature:
       - core:
           csit-list: >
index 38b6f95a5fc4fd236388845b1a08a4d9e3105506..3f5581c94c4fd28ea18731920311d1d4c9d261ae 100644 (file)
@@ -22,6 +22,8 @@
           branch: "stable/aluminium"
           jre: "openjdk11"
 
+    karaf-version: netconf
+
     install:
       - only:
           scope: "only"
@@ -74,6 +76,8 @@
             \ INIT_DEVICE_COUNT:1000 -v MAX_DEVICE_COUNT:3000 -v DEVICE_INCREMENT:500\
             \ -v TESTTOOL_BOOT_TIMEOUT:600s -v MAX_HEAP:4G --flattenkeywords name:*repeat*keyword*"
 
+    karaf-version: netconf
+
     install:
       - only:
           scope: "only"
index 1e4d30b016282d9a69703bbd2591ee10a760dece..66eab97dd35a77b49f68403e2812bbde0b50dd6f 100644 (file)
@@ -28,6 +28,8 @@
           branch: "stable/aluminium"
           jre: "openjdk11"
 
+    karaf-version: netconf
+
     install:
       - all:
           scope: "all"
@@ -39,7 +41,6 @@
       odl-restconf,
       odl-netconf-mdsal,
       odl-mdsal-apidocs,
-      odl-mdsal-model-inventory,
       odl-clustering-test-app,
       odl-netconf-topology