jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -ex
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="${MAVEN_PARAMS} \
   9     --show-version \
  10     --batch-mode \
  11     -Djenkins \
  12     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  13     -Dmaven.repo.local=/tmp/r \
  14     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
  15
  16 rm -rf "$BUILD_DIR"
  17 mkdir -p "$BUILD_DIR"
  18 cd "$BUILD_DIR"
  19
  20 # Download distribution pom.xml
  21 wget "https://git.opendaylight.org/gerrit/gitweb?p=integration/distribution.git;a=blob_plain;f=artifacts/upstream/properties/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
  22 cat pom.xml
  23
  24 # Set up git committer name and email, needed for commit creation when cherry-picking.
  25 export EMAIL="jenkins-$SILO@opendaylight.org"
  26 export GIT_COMMITTER_NAME="Multipatch Job"
  27
  28 # Extract a list of patches per project from an comment trigger. An example is:
  29 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  30 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  31     # Decode Base64 before parsing text.
  32     GERRIT_EVENT_COMMENT_TEXT=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | base64 -d)
  33
  34     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  35         BUILD_FAST="true"
  36         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  37     else
  38         BUILD_FAST="false"
  39         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  40     fi
  41     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  42 fi
  43 # check if topic exists:
  44 # if topic=binding-rpc, then checkout first patch in binding-rpc topic (if it exists)
  45 # if topic:binding-rpc, then cherry-pick first patch in binding-rpc topic (if it exists)
  46 if [[ "${PATCHES_TO_BUILD}" == *"topic"* ]]; then
  47     if [[ "${PATCHES_TO_BUILD}" == *"topic="* ]]; then
  48         CHERRY_PICK="false"
  49         TOPIC="${PATCHES_TO_BUILD#*=}"
  50     elif [[ "${PATCHES_TO_BUILD}" == *"topic:"* ]]; then
  51         CHERRY_PICK="true"
  52         TOPIC="${PATCHES_TO_BUILD#*:}"
  53     else
  54         echo "ERROR: Topic has wrong format" && exit 1
  55     fi
  56     echo "Create topic ${TOPIC} patch list"
  57     PATCHES_TO_BUILD=""
  58     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  59     echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
  60     for PROJECT in "${PROJECT_LIST[@]}"; do
  61         # get all patches number for a topic for a given project
  62         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 "jenkins-$SILO@git.opendaylight.org" gerrit query status:open "topic:${TOPIC}" "project:${PROJECT}" 2> /dev/null \
  63         | grep 'number:' | awk '{{ print $2 }}')" || true
  64         # add project if it is the first with patches or it is not the first
  65         if [[ -z "${PATCHES_TO_BUILD}" && -n "${GERRIT_PATCH_LIST[*]}" ]]; then
  66             PATCHES_TO_BUILD="${PROJECT}"
  67         elif [[ -n "${PATCHES_TO_BUILD}" ]]; then
  68             if [[ -n "${GERRIT_PATCH_LIST[*]}" ]]; then
  69                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  70             fi
  71             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  72         fi
  73         # sort project patches
  74         if [[ -n "${GERRIT_PATCH_LIST[*]}" ]]; then
  75             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  76             REF_LIST=()
  77             # create reference list with patch number-refspec
  78             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  79                 REFSPEC=$(ssh -p 29418 "jenkins-$SILO@git.opendaylight.org" gerrit query "change:${PATCH}" --current-patch-set \
  80                 | grep 'ref:' | awk '{{ print $2 }}')
  81                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  82             done
  83             # sort reference list by patch number
  84             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  85             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  86             # add refspec to patches to build list
  87             COUNT=0
  88             for PATCH in "${SORT_REF_LIST[@]}"; do
  89                 COUNT=$((COUNT+1))
  90                 if [ "${COUNT}" == "1" ] && [ "${CHERRY_PICK}" == "false" ]; then
  91                     # checkout patch
  92                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}=${PATCH/*-/}"
  93                 else
  94                     # cherry-pick is better than checkout patch
  95                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  96                 fi
  97             done
  98         fi
  99     done
 100 fi
 101 echo "Patches to build: ${PATCHES_TO_BUILD}"
 102 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
 103
 104 # First phase: clone the necessary repos and set the patches up
 105
 106 declare -a PROJECTS
 107
 108 # For each patch:
 109 # 1. Clone the project.
 110 # 2. Checkout an specific (typically unmerged) Gerrit patch. If none,
 111 # use distribution pom.xml file to figure out right branch or tag to checkout.
 112 # In case of Gerrit patch in MRI project, adjust version for the stream.
 113 # 3. Optionally, cherry-pick series of patches on top of the checkout.
 114 #
 115 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 116 # list of project[=checkout][:cherry-pick]* values. Examples:
 117 #
 118 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 119 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 120 #
 121 # Checkout patches for both odlparent and yangtools:
 122 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 123 #
 124 # Checkout a patch for controller, cherry-pick another patch on top of it:
 125 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 126 distribution_status="not_included"
 127 for patch in "${PATCHES[@]}"
 128 do
 129     echo "-- working on ${patch} --"
 130     # For patch=controller=61/29761/5:45/29645/6, this gives controller.
 131     PROJECT="$(echo "${patch}" | cut -d':' -f 1 | cut -d'=' -f 1)"
 132     if [ "${PROJECT}" == "integration/distribution" ]; then
 133         distribution_status="included"
 134     fi
 135     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 136     PROJECTS+=("${PROJECT_SHORTNAME}")
 137     echo "1. cloning project ${PROJECT}"
 138     git clone "https://git.opendaylight.org/gerrit/${PROJECT}"
 139     cd "${PROJECT_SHORTNAME}"
 140     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5.
 141     CHECKOUT="$(echo "${patch}" | cut -d'=' -s -f 2 | cut -d':' -f 1)"
 142     # If there is a base patch for this project, checkout patch, otherwise use
 143     # distribution pom.xml file to figure out right branch or tag to checkout.
 144     if [ "x${CHECKOUT}" != "x" ]; then
 145         echo "2. checking out patch ${CHECKOUT}"
 146         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 147         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 148         git checkout FETCH_HEAD
 149         # If the patch is for MRI project, adjust the MRI versions
 150         # shellcheck disable=SC2235
 151         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
 152             ODLPARENT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:odlparent.version ../pom.xml)"
 153             echo "change odlparent version to ${ODLPARENT_VERSION}"
 154             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
 155             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
 156         fi
 157         # shellcheck disable=SC2235
 158         if [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
 159             YANGTOOLS_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:yangtools.version ../pom.xml)"
 160             echo "change yangtools version to ${YANGTOOLS_VERSION}"
 161             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
 162             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
 163         fi
 164         # shellcheck disable=SC2235
 165         if [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
 166             MDSAL_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:mdsal.version ../pom.xml)"
 167             echo "change mdsal version to ${MDSAL_VERSION}"
 168             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
 169             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
 170         fi
 171         if [ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]; then
 172             CONTROLLER_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:controller.version ../pom.xml)"
 173             echo "change controller version to ${CONTROLLER_VERSION}"
 174             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.controller\"\] -v "${CONTROLLER_VERSION}"
 175             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.controller\"\] -v "${CONTROLLER_VERSION}"
 176         fi
 177     else
 178         # If project with no patch is MRI, download release tag:
 179         # shellcheck disable=SC2235
 180         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || [ "${PROJECT}" == "mdsal" ] || ([ "${PROJECT}" == "controller" ] && [ "${DISTROSTREAM}" != "magnesium" ]); then
 181             # shellcheck disable=SC2086
 182             PROJECT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:${PROJECT_SHORTNAME}.version ../pom.xml)"
 183             echo "2. checking out tag v${PROJECT_VERSION}"
 184             git checkout "tags/v${PROJECT_VERSION}"
 185         # Otherwise download distribution branch:
 186         else
 187             echo "2. checking out branch ${DISTRIBUTION_BRANCH_TO_BUILD}"
 188             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 189         fi
 190     fi
 191     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 192     PICK_SEGMENT="$(echo "${patch}" | cut -d: -s -f 2-)"
 193     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 194     for pick in "${PICKS[@]}"
 195     do
 196         echo "3. cherry-picking ${pick}"
 197         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 198         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 199     done
 200     cd "${BUILD_DIR}"
 201 done
 202
 203 # Finally add distribution if there is no int/dist patch
 204 if [ "${distribution_status}" == "not_included" ]; then
 205     echo "adding integration/distribution"
 206     PROJECTS+=(distribution)
 207     # clone distribution and add it as a module in root pom
 208     git clone "https://git.opendaylight.org/gerrit/integration/distribution"
 209     cd distribution
 210     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 211     cd "${BUILD_DIR}"
 212 fi
 213
 214 # Second phase: build everything
 215
 216 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 217     # Set Fast build if project is not in BUILD_NORMAL and BUILD_FAST is true
 218     if [[ "${BUILD_NORMAL}" != *"${PROJECT_SHORTNAME}"* ]] && ${BUILD_FAST}; then
 219         fast_option="-Pq"
 220     else
 221         fast_option=""
 222     fi
 223     pushd "${PROJECT_SHORTNAME}"
 224         # Build project
 225         # shellcheck disable=SC2086
 226         "$MVN" clean install \
 227             -e ${fast_option} \
 228             -Dstream="$DISTROSTREAM" \
 229             -Dgitid.skip=false \
 230             -Dmaven.gitcommitid.skip=false \
 231             --global-settings "$GLOBAL_SETTINGS_FILE" \
 232             --settings "$SETTINGS_FILE" \
 233             $MAVEN_OPTIONS
 234         # Since we've installed the artifacts, we can clean the build and save disk space
 235         # shellcheck disable=SC2086
 236         "$MVN" clean \
 237             -e ${fast_option} \
 238             -Dstream="$DISTROSTREAM" \
 239             -Dgitid.skip=false \
 240             -Dmaven.gitcommitid.skip=false \
 241             --global-settings "$GLOBAL_SETTINGS_FILE" \
 242             --settings "$SETTINGS_FILE" \
 243             $MAVEN_OPTIONS
 244     popd
 245 done