X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Fintegration%2Fmultipatch-distribution.sh;h=eac630698ebbee1b5c047085159e3a2489e55868;hb=HEAD;hp=cd7fbbeb62bc6b3bbec44f89b0594578aa867a0b;hpb=a35ac65a18edd8af0638c8234e9d60857b3cee13;p=releng%2Fbuilder.git

diff --git a/jjb/integration/multipatch-distribution.sh b/jjb/integration/multipatch-distribution.sh
index cd7fbbeb6..92cc1e1c3 100755
--- a/jjb/integration/multipatch-distribution.sh
+++ b/jjb/integration/multipatch-distribution.sh
@@ -1,38 +1,36 @@
 #!/bin/bash
 
-set -e
-
-# TODO: 1) clean up inline todo's below :)
-# TODO: 2) Use just a topic branch to create a distribution.  see this email:
-#          https://lists.opendaylight.org/pipermail/discuss/2015-December/006040.html
-# TODO: 3) Bubble up CSIT jobs that are triggered by the multipatch job to a jenkins
-#          parameter.  the default can be distribution-test which calls all CSIT jobs
-#          but being able to easily override it to a smaller subset (or none) will be
-#          helpful
+set -ex
 
 # create a fresh empty place to build this custom distribution
 BUILD_DIR=${WORKSPACE}/patch_tester
 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
-MAVEN_OPTIONS="$(echo --show-version \
+MAVEN_OPTIONS="${MAVEN_PARAMS} \
+    --show-version \
     --batch-mode \
     -Djenkins \
     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
     -Dmaven.repo.local=/tmp/r \
-    -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)"
+    -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
+
+rm -rf "$BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
 
-rm -rf $BUILD_DIR
-mkdir -p $BUILD_DIR
-cd $BUILD_DIR
+# Download distribution pom.xml
+wget "https://git.opendaylight.org/gerrit/gitweb?p=integration/distribution.git;a=blob_plain;f=artifacts/upstream/properties/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
+cat pom.xml
 
 # Set up git committer name and email, needed for commit creation when cherry-picking.
-export EMAIL="sandbox@jenkins.opendaylight.org"
+export EMAIL="jenkins-$SILO@opendaylight.org"
 export GIT_COMMITTER_NAME="Multipatch Job"
 
 # Extract a list of patches per project from an comment trigger. An example is:
-# Patch Set 1:
-#
 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
+    # Decode Base64 before parsing text.
+    GERRIT_EVENT_COMMENT_TEXT=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | base64 -d)
+
     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
         BUILD_FAST="true"
         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
@@ -42,34 +40,43 @@ if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
     fi
     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
 fi
-if ${BUILD_FAST}; then
-    fast_option="-Pq"
-else
-    fast_option=""
-fi
-# check if topic exists, e.g. topic=binding-tlc-rpc
-if [[ "${PATCHES_TO_BUILD}" == *topic* ]]; then
-    TOPIC="${PATCHES_TO_BUILD#*=}"
+# check if topic exists:
+# if topic=binding-rpc, then checkout first patch in binding-rpc topic (if it exists)
+# if topic:binding-rpc, then cherry-pick first patch in binding-rpc topic (if it exists)
+if [[ "${PATCHES_TO_BUILD}" == *"topic"* ]]; then
+    if [[ "${PATCHES_TO_BUILD}" == *"topic="* ]]; then
+        CHERRY_PICK="false"
+        TOPIC="${PATCHES_TO_BUILD#*=}"
+    elif [[ "${PATCHES_TO_BUILD}" == *"topic:"* ]]; then
+        CHERRY_PICK="true"
+        TOPIC="${PATCHES_TO_BUILD#*:}"
+    else
+        echo "ERROR: Topic has wrong format" && exit 1
+    fi
     echo "Create topic ${TOPIC} patch list"
     PATCHES_TO_BUILD=""
     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
+    echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
     for PROJECT in "${PROJECT_LIST[@]}"; do
         # get all patches number for a topic for a given project
-        IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} \
+        IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 "jenkins-$SILO@git.opendaylight.org" gerrit query status:open "topic:${TOPIC}" "project:${PROJECT}" 2> /dev/null \
         | grep 'number:' | awk '{{ print $2 }}')" || true
-        echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
         # add project if it is the first with patches or it is not the first
-        if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
+        if [[ -z "${PATCHES_TO_BUILD}" && -n "${GERRIT_PATCH_LIST[*]}" ]]; then
             PATCHES_TO_BUILD="${PROJECT}"
-        elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
+        elif [[ -n "${PATCHES_TO_BUILD}" ]]; then
+            if [[ -n "${GERRIT_PATCH_LIST[*]}" ]]; then
+                echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
+            fi
             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
         fi
         # sort project patches
-        if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
+        if [[ -n "${GERRIT_PATCH_LIST[*]}" ]]; then
+            echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
             REF_LIST=()
             # create reference list with patch number-refspec
             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
-                REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
+                REFSPEC=$(ssh -p 29418 "jenkins-$SILO@git.opendaylight.org" gerrit query "change:${PATCH}" --current-patch-set \
                 | grep 'ref:' | awk '{{ print $2 }}')
                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
             done
@@ -77,13 +84,20 @@ if [[ "${PATCHES_TO_BUILD}" == *topic* ]]; then
             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
             # add refspec to patches to build list
+            COUNT=0
             for PATCH in "${SORT_REF_LIST[@]}"; do
-                PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
+                COUNT=$((COUNT+1))
+                if [ "${COUNT}" == "1" ] && [ "${CHERRY_PICK}" == "false" ]; then
+                    # checkout patch
+                    PATCHES_TO_BUILD="${PATCHES_TO_BUILD}=${PATCH/*-/}"
+                else
+                    # cherry-pick is better than checkout patch
+                    PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
+                fi
             done
         fi
     done
 fi
-
 echo "Patches to build: ${PATCHES_TO_BUILD}"
 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
 
@@ -92,79 +106,106 @@ IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
 declare -a PROJECTS
 
 # For each patch:
-# * Clone the project.
-# * Optionally, checkout a specific (typically unmerged) Gerrit patch. If none,
-#   default to Integration/Distribution branch via {branch} JJB param.
-# * Also optionally, cherry-pick series of patches on top of the checkout.
-# * Final option: perform a 'release' by removing "-SNAPSHOT" everywhere within the project.
+# 1. Clone the project.
+# 2. Checkout an specific (typically unmerged) Gerrit patch. If none,
+# use distribution pom.xml file to figure out right branch or tag to checkout.
+# In case of Gerrit patch in MRI project, adjust version for the stream.
+# 3. Optionally, cherry-pick series of patches on top of the checkout.
 #
 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
-# list of project[=checkout][:cherry-pick]* values.
+# list of project[=checkout][:cherry-pick]* values. Examples:
 #
 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
-#
 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 #
 # Checkout patches for both odlparent and yangtools:
-#
 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 #
 # Checkout a patch for controller, cherry-pick another patch on top of it:
-#
 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 distribution_status="not_included"
-for proto_patch in "${PATCHES[@]}"
+for patch in "${PATCHES[@]}"
 do
-    echo "working on ${proto_patch}"
-    # For patch=controller=61/29761/5:45/29645/6, this gives controller
-    PROJECT="$(echo ${proto_patch} | cut -d\: -f 1 | cut -d\= -f 1)"
+    echo "-- working on ${patch} --"
+    # For patch=controller=61/29761/5:45/29645/6, this gives controller.
+    PROJECT="$(echo "${patch}" | cut -d':' -f 1 | cut -d'=' -f 1)"
     if [ "${PROJECT}" == "integration/distribution" ]; then
         distribution_status="included"
     fi
     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
     PROJECTS+=("${PROJECT_SHORTNAME}")
-    echo "cloning project ${PROJECT}"
-    git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
-    cd ${PROJECT_SHORTNAME}
-    if [ "$(echo -n ${proto_patch} | tail -c 1)" == 'r' ]; then
-        pure_patch="$(echo -n $proto_patch | head -c -1)"
-    else
-        pure_patch="$proto_patch"
-    fi
-    # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5
-    CHECKOUT="$(echo ${pure_patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
+    echo "1. cloning project ${PROJECT}"
+    git clone "https://git.opendaylight.org/gerrit/${PROJECT}"
+    cd "${PROJECT_SHORTNAME}"
+    # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5.
+    CHECKOUT="$(echo "${patch}" | cut -d'=' -s -f 2 | cut -d':' -f 1)"
+    # If there is a base patch for this project, checkout patch, otherwise use
+    # distribution pom.xml file to figure out right branch or tag to checkout.
     if [ "x${CHECKOUT}" != "x" ]; then
-        echo "checking out ${CHECKOUT}"
+        echo "2. checking out patch ${CHECKOUT}"
         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
         git checkout FETCH_HEAD
+        # If the patch is for MRI project, adjust the MRI versions
+        # shellcheck disable=SC2235
+        if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
+            ODLPARENT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:odlparent.version ../pom.xml)"
+            echo "change odlparent version to ${ODLPARENT_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
+        fi
+        # shellcheck disable=SC2235
+        if [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
+            YANGTOOLS_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:yangtools.version ../pom.xml)"
+            echo "change yangtools version to ${YANGTOOLS_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
+        fi
+	# shellcheck disable=SC2235
+        if [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
+            MDSAL_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:mdsal.version ../pom.xml)"
+            echo "change mdsal version to ${MDSAL_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
+        fi
+        if [ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]; then
+            CONTROLLER_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:controller.version ../pom.xml)"
+            echo "change controller version to ${CONTROLLER_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.controller\"\] -v "${CONTROLLER_VERSION}"
+            find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.controller\"\] -v "${CONTROLLER_VERSION}"
+        fi
     else
-        echo "checking out ${DISTRIBUTION_BRANCH_TO_BUILD}"
-        git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
+        # If project with no patch is MRI, download release tag:
+        # shellcheck disable=SC2235
+        if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
+            # shellcheck disable=SC2086
+            PROJECT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:${PROJECT_SHORTNAME}.version ../pom.xml)"
+            echo "2. checking out tag v${PROJECT_VERSION}"
+            git checkout "tags/v${PROJECT_VERSION}"
+        # Otherwise download distribution branch:
+        else
+            echo "2. checking out branch ${DISTRIBUTION_BRANCH_TO_BUILD}"
+            git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
+        fi
     fi
     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
-    PICK_SEGMENT="$(echo "${pure_patch}" | cut -d\: -s -f 2-)"
+    PICK_SEGMENT="$(echo "${patch}" | cut -d: -s -f 2-)"
     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
     for pick in "${PICKS[@]}"
     do
-        echo "cherry-picking ${pick}"
+        echo "3. cherry-picking ${pick}"
         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
     done
-    # Check whether the patch ends with 'r', and the patch isn't the project
-    # (to avoid releasing controller...)
-    if [ "${proto_patch: -1}" = "r" ] && [ "${proto_patch}" != "${PROJECT}" ]; then
-        # Here 'r' means release. Useful for testing Nitrogen Odlparent changes.
-        find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
-    fi
     cd "${BUILD_DIR}"
 done
 
+# Finally add distribution if there is no int/dist patch
 if [ "${distribution_status}" == "not_included" ]; then
     echo "adding integration/distribution"
     PROJECTS+=(distribution)
     # clone distribution and add it as a module in root pom
-    git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
+    git clone "https://git.opendaylight.org/gerrit/integration/distribution"
     cd distribution
     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
     cd "${BUILD_DIR}"
@@ -173,26 +214,32 @@ fi
 # Second phase: build everything
 
 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
+    # Set Fast build if project is not in BUILD_NORMAL and BUILD_FAST is true
+    if [[ "${BUILD_NORMAL}" != *"${PROJECT_SHORTNAME}"* ]] && ${BUILD_FAST}; then
+        fast_option="-Pq"
+    else
+        fast_option=""
+    fi
     pushd "${PROJECT_SHORTNAME}"
-    # Build project
-    "$MVN" clean install \
-    -e ${fast_option} \
-    -Dstream="$DISTROSTREAM" \
-    -Dgitid.skip=false \
-    -Dmaven.gitcommitid.skip=false \
-    --global-settings "$GLOBAL_SETTINGS_FILE" \
-    --settings "$SETTINGS_FILE" \
-    $MAVEN_OPTIONS
-    # Since we've installed the artifacts, we can clean the build and save
-    # disk space
-    "$MVN" clean \
-    -e ${fast_option} \
-    -Dstream="$DISTROSTREAM" \
-    -Dgitid.skip=false \
-    -Dmaven.gitcommitid.skip=false \
-    --global-settings "$GLOBAL_SETTINGS_FILE" \
-    --settings "$SETTINGS_FILE" \
-    $MAVEN_OPTIONS
+        # Build project
+        # shellcheck disable=SC2086
+        "$MVN" clean install \
+            -e ${fast_option} \
+            -Dstream="$DISTROSTREAM" \
+            -Dgitid.skip=false \
+            -Dmaven.gitcommitid.skip=false \
+            --global-settings "$GLOBAL_SETTINGS_FILE" \
+            --settings "$SETTINGS_FILE" \
+            $MAVEN_OPTIONS
+        # Since we've installed the artifacts, we can clean the build and save disk space
+        # shellcheck disable=SC2086
+        "$MVN" clean \
+            -e ${fast_option} \
+            -Dstream="$DISTROSTREAM" \
+            -Dgitid.skip=false \
+            -Dmaven.gitcommitid.skip=false \
+            --global-settings "$GLOBAL_SETTINGS_FILE" \
+            --settings "$SETTINGS_FILE" \
+            $MAVEN_OPTIONS
     popd
 done
-