jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -ex
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="${MAVEN_PARAMS} \
   9     --show-version \
  10     --batch-mode \
  11     -Djenkins \
  12     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  13     -Dmaven.repo.local=/tmp/r \
  14     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
  15
  16 rm -rf "$BUILD_DIR"
  17 mkdir -p "$BUILD_DIR"
  18 cd "$BUILD_DIR"
  19
  20 # Download distribution pom.xml
  21 wget "https://git.opendaylight.org/gerrit/gitweb?p=integration/distribution.git;a=blob_plain;f=artifacts/upstream/properties/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
  22 cat pom.xml
  23
  24 # Set up git committer name and email, needed for commit creation when cherry-picking.
  25 export EMAIL="jenkins-$SILO@opendaylight.org"
  26 export GIT_COMMITTER_NAME="Multipatch Job"
  27
  28 # Extract a list of patches per project from an comment trigger. An example is:
  29 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  30 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  31     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  32         BUILD_FAST="true"
  33         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  34     else
  35         BUILD_FAST="false"
  36         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  37     fi
  38     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  39 fi
  40 # check if topic exists:
  41 # if topic=binding-rpc, then checkout first patch in binding-rpc topic (if it exists)
  42 # if topic:binding-rpc, then cherry-pick first patch in binding-rpc topic (if it exists)
  43 if [[ "${PATCHES_TO_BUILD}" == *"topic"* ]]; then
  44     if [[ "${PATCHES_TO_BUILD}" == *"topic="* ]]; then
  45         CHERRY_PICK="false"
  46         TOPIC="${PATCHES_TO_BUILD#*=}"
  47     elif [[ "${PATCHES_TO_BUILD}" == *"topic:"* ]]; then
  48         CHERRY_PICK="true"
  49         TOPIC="${PATCHES_TO_BUILD#*:}"
  50     else
  51         echo "ERROR: Topic has wrong format" && exit 1
  52     fi
  53     echo "Create topic ${TOPIC} patch list"
  54     PATCHES_TO_BUILD=""
  55     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  56     echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
  57     for PROJECT in "${PROJECT_LIST[@]}"; do
  58         # get all patches number for a topic for a given project
  59         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 "jenkins-$SILO@git.opendaylight.org" gerrit query status:open "topic:${TOPIC}" "project:${PROJECT}" 2> /dev/null \
  60         | grep 'number:' | awk '{{ print $2 }}')" || true
  61         # add project if it is the first with patches or it is not the first
  62         if [[ -z "${PATCHES_TO_BUILD}" && -n "${GERRIT_PATCH_LIST[*]}" ]]; then
  63             PATCHES_TO_BUILD="${PROJECT}"
  64         elif [[ -n "${PATCHES_TO_BUILD}" ]]; then
  65             if [[ -n "${GERRIT_PATCH_LIST[*]}" ]]; then
  66                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  67             fi
  68             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  69         fi
  70         # sort project patches
  71         if [[ -n "${GERRIT_PATCH_LIST[*]}" ]]; then
  72             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  73             REF_LIST=()
  74             # create reference list with patch number-refspec
  75             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  76                 REFSPEC=$(ssh -p 29418 "jenkins-$SILO@git.opendaylight.org" gerrit query "change:${PATCH}" --current-patch-set \
  77                 | grep 'ref:' | awk '{{ print $2 }}')
  78                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  79             done
  80             # sort reference list by patch number
  81             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  82             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  83             # add refspec to patches to build list
  84             COUNT=0
  85             for PATCH in "${SORT_REF_LIST[@]}"; do
  86                 COUNT=$((COUNT+1))
  87                 if [ "${COUNT}" == "1" ] && [ "${CHERRY_PICK}" == "false" ]; then
  88                     # checkout patch
  89                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}=${PATCH/*-/}"
  90                 else
  91                     # cherry-pick is better than checkout patch
  92                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  93                 fi
  94             done
  95         fi
  96     done
  97 fi
  98 echo "Patches to build: ${PATCHES_TO_BUILD}"
  99 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
 100
 101 # First phase: clone the necessary repos and set the patches up
 102
 103 declare -a PROJECTS
 104
 105 # For each patch:
 106 # 1. Clone the project.
 107 # 2. Checkout an specific (typically unmerged) Gerrit patch. If none,
 108 # use distribution pom.xml file to figure out right branch or tag to checkout.
 109 # In case of Gerrit patch in MRI project, adjust version for the stream.
 110 # 3. Optionally, cherry-pick series of patches on top of the checkout.
 111 #
 112 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 113 # list of project[=checkout][:cherry-pick]* values. Examples:
 114 #
 115 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 116 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 117 #
 118 # Checkout patches for both odlparent and yangtools:
 119 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 120 #
 121 # Checkout a patch for controller, cherry-pick another patch on top of it:
 122 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 123 distribution_status="not_included"
 124 for patch in "${PATCHES[@]}"
 125 do
 126     echo "-- working on ${patch} --"
 127     # For patch=controller=61/29761/5:45/29645/6, this gives controller.
 128     PROJECT="$(echo "${patch}" | cut -d':' -f 1 | cut -d'=' -f 1)"
 129     if [ "${PROJECT}" == "integration/distribution" ]; then
 130         distribution_status="included"
 131     fi
 132     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 133     PROJECTS+=("${PROJECT_SHORTNAME}")
 134     echo "1. cloning project ${PROJECT}"
 135     git clone "https://git.opendaylight.org/gerrit/${PROJECT}"
 136     cd "${PROJECT_SHORTNAME}"
 137     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5.
 138     CHECKOUT="$(echo "${patch}" | cut -d'=' -s -f 2 | cut -d':' -f 1)"
 139     # If there is a base patch for this project, checkout patch, otherwise use
 140     # distribution pom.xml file to figure out right branch or tag to checkout.
 141     if [ "x${CHECKOUT}" != "x" ]; then
 142         echo "2. checking out patch ${CHECKOUT}"
 143         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 144         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 145         git checkout FETCH_HEAD
 146         # If the patch is for MRI project, adjust the MRI versions
 147         # shellcheck disable=SC2235
 148         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ] && [ "${DISTROSTREAM}" != "sodium" ]); then
 149             ODLPARENT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:odlparent.version ../pom.xml)"
 150             echo "change odlparent version to ${ODLPARENT_VERSION}"
 151             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
 152             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
 153         fi
 154         # shellcheck disable=SC2235
 155         if [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ] && [ "${DISTROSTREAM}" != "sodium" ]); then
 156             YANGTOOLS_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:yangtools.version ../pom.xml)"
 157             echo "change yangtools version to ${YANGTOOLS_VERSION}"
 158             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
 159             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
 160         fi
 161         # shellcheck disable=SC2235
 162         if [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ] && [ "${DISTROSTREAM}" != "sodium" ]); then
 163             MDSAL_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:mdsal.version ../pom.xml)"
 164             echo "change mdsal version to ${MDSAL_VERSION}"
 165             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
 166             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
 167         fi
 168         if [ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ] && [ "${DISTROSTREAM}" != "sodium" ]; then
 169             CONTROLLER_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:controller.version ../pom.xml)"
 170             echo "change controller version to ${CONTROLLER_VERSION}"
 171             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.controller\"\] -v "${CONTROLLER_VERSION}"
 172             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.controller\"\] -v "${CONTROLLER_VERSION}"
 173         fi
 174     else
 175         # If project with no patch is MRI, download release tag:
 176         # shellcheck disable=SC2235
 177         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ] && [ "${DISTROSTREAM}" != "sodium" ]); then
 178             # shellcheck disable=SC2086
 179             PROJECT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:${PROJECT_SHORTNAME}.version ../pom.xml)"
 180             echo "2. checking out tag v${PROJECT_VERSION}"
 181             git checkout "tags/v${PROJECT_VERSION}"
 182         # Otherwise download distribution branch:
 183         else
 184             echo "2. checking out branch ${DISTRIBUTION_BRANCH_TO_BUILD}"
 185             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 186         fi
 187     fi
 188     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 189     PICK_SEGMENT="$(echo "${patch}" | cut -d: -s -f 2-)"
 190     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 191     for pick in "${PICKS[@]}"
 192     do
 193         echo "3. cherry-picking ${pick}"
 194         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 195         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 196     done
 197     cd "${BUILD_DIR}"
 198 done
 199
 200 # Finally add distribution if there is no int/dist patch
 201 if [ "${distribution_status}" == "not_included" ]; then
 202     echo "adding integration/distribution"
 203     PROJECTS+=(distribution)
 204     # clone distribution and add it as a module in root pom
 205     git clone "https://git.opendaylight.org/gerrit/integration/distribution"
 206     cd distribution
 207     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 208     cd "${BUILD_DIR}"
 209 fi
 210
 211 # Second phase: build everything
 212
 213 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 214     # Set Fast build if project is not in BUILD_NORMAL and BUILD_FAST is true
 215     if [[ "${BUILD_NORMAL}" != *"${PROJECT_SHORTNAME}"* ]] && ${BUILD_FAST}; then
 216         fast_option="-Pq"
 217     else
 218         fast_option=""
 219     fi
 220     pushd "${PROJECT_SHORTNAME}"
 221         # Build project
 222         # shellcheck disable=SC2086
 223         "$MVN" clean install \
 224             -e ${fast_option} \
 225             -Dstream="$DISTROSTREAM" \
 226             -Dgitid.skip=false \
 227             -Dmaven.gitcommitid.skip=false \
 228             --global-settings "$GLOBAL_SETTINGS_FILE" \
 229             --settings "$SETTINGS_FILE" \
 230             $MAVEN_OPTIONS
 231         # Since we've installed the artifacts, we can clean the build and save disk space
 232         # shellcheck disable=SC2086
 233         "$MVN" clean \
 234             -e ${fast_option} \
 235             -Dstream="$DISTROSTREAM" \
 236             -Dgitid.skip=false \
 237             -Dmaven.gitcommitid.skip=false \
 238             --global-settings "$GLOBAL_SETTINGS_FILE" \
 239             --settings "$SETTINGS_FILE" \
 240             $MAVEN_OPTIONS
 241     popd
 242 done
 243