jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -e
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="${MAVEN_PARAMS} \
   9     --show-version \
  10     --batch-mode \
  11     -Djenkins \
  12     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  13     -Dmaven.repo.local=/tmp/r \
  14     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
  15
  16 rm -rf $BUILD_DIR
  17 mkdir -p $BUILD_DIR
  18 cd $BUILD_DIR
  19
  20 # Download distribution pom.xml
  21 wget "http://git.opendaylight.org/gerrit/gitweb?p=integration/distribution.git;a=blob_plain;f=artifacts/upstream/properties/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
  22 cat pom.xml
  23
  24 # Set up git committer name and email, needed for commit creation when cherry-picking.
  25 export EMAIL="sandbox@jenkins.opendaylight.org"
  26 export GIT_COMMITTER_NAME="Multipatch Job"
  27
  28 # Extract a list of patches per project from an comment trigger. An example is:
  29 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  30 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  31     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  32         BUILD_FAST="true"
  33         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  34     else
  35         BUILD_FAST="false"
  36         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  37     fi
  38     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  39 fi
  40 # check if topic exists:
  41 # if topic=binding-rpc, then checkout first patch in binding-rpc topic (if it exists)
  42 # if topic:binding-rpc, then cherry-pick first patch in binding-rpc topic (if it exists)
  43 if [[ "${PATCHES_TO_BUILD}" == *"topic"* ]]; then
  44     if [[ "${PATCHES_TO_BUILD}" == *"topic="* ]]; then
  45         CHERRY_PICK="false"
  46         TOPIC="${PATCHES_TO_BUILD#*=}"
  47     elif [[ "${PATCHES_TO_BUILD}" == *"topic:"* ]]; then
  48         CHERRY_PICK="true"
  49         TOPIC="${PATCHES_TO_BUILD#*:}"
  50     else
  51         echo "ERROR: Topic has wrong format" && exit 1
  52     fi
  53     echo "Create topic ${TOPIC} patch list"
  54     PATCHES_TO_BUILD=""
  55     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  56     echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
  57     for PROJECT in "${PROJECT_LIST[@]}"; do
  58         # get all patches number for a topic for a given project
  59         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} 2> /dev/null \
  60         | grep 'number:' | awk '{{ print $2 }}')" || true
  61         # add project if it is the first with patches or it is not the first
  62         if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  63             PATCHES_TO_BUILD="${PROJECT}"
  64         elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
  65             if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  66                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  67             fi
  68             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  69         fi
  70         # sort project patches
  71         if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  72             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  73             REF_LIST=()
  74             # create reference list with patch number-refspec
  75             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  76                 REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
  77                 | grep 'ref:' | awk '{{ print $2 }}')
  78                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  79             done
  80             # sort reference list by patch number
  81             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  82             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  83             # add refspec to patches to build list
  84             COUNT=0
  85             for PATCH in "${SORT_REF_LIST[@]}"; do
  86                 COUNT=$((COUNT+1))
  87                 if [ "${COUNT}" == "1" ] && [ "${CHERRY_PICK}" == "false" ]; then
  88                     # checkout patch
  89                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}=${PATCH/*-/}"
  90                 else
  91                     # cherry-pick is better than checkout patch
  92                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  93                 fi
  94             done
  95         fi
  96     done
  97 fi
  98 echo "Patches to build: ${PATCHES_TO_BUILD}"
  99 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
 100
 101 # First phase: clone the necessary repos and set the patches up
 102
 103 declare -a PROJECTS
 104
 105 # For each patch:
 106 # 1. Clone the project.
 107 # 2. Checkout an specific (typically unmerged) Gerrit patch. If none,
 108 # use distribution pom.xml file to figure out right branch or tag to checkout.
 109 # In case of Gerrit patch in MRI project, adjust version for the stream.
 110 # 3. Optionally, cherry-pick series of patches on top of the checkout.
 111 #
 112 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 113 # list of project[=checkout][:cherry-pick]* values. Examples:
 114 #
 115 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 116 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 117 #
 118 # Checkout patches for both odlparent and yangtools:
 119 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 120 #
 121 # Checkout a patch for controller, cherry-pick another patch on top of it:
 122 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 123 distribution_status="not_included"
 124 for patch in "${PATCHES[@]}"
 125 do
 126     echo "-- working on ${patch} --"
 127     # For patch=controller=61/29761/5:45/29645/6, this gives controller.
 128     PROJECT="$(echo ${patch} | cut -d\: -f 1 | cut -d\= -f 1)"
 129     if [ "${PROJECT}" == "integration/distribution" ]; then
 130         distribution_status="included"
 131     fi
 132     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 133     PROJECTS+=("${PROJECT_SHORTNAME}")
 134     echo "1. cloning project ${PROJECT}"
 135     git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
 136     cd ${PROJECT_SHORTNAME}
 137     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5.
 138     CHECKOUT="$(echo ${patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
 139     # If there is a base patch for this project, checkout patch, otherwise use
 140     # distribution pom.xml file to figure out right branch or tag to checkout.
 141     if [ "x${CHECKOUT}" != "x" ]; then
 142         echo "2. checking out patch ${CHECKOUT}"
 143         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 144         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 145         git checkout FETCH_HEAD
 146         # If the patch is for MRI project, adjust the MRI versions
 147         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 148             ODLPARENT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:odlparent.version ../pom.xml)"
 149             echo "change odlparent version to ${ODLPARENT_VERSION}"
 150             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
 151             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}"
 152         fi
 153         if [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 154             YANGTOOLS_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:yangtools.version ../pom.xml)"
 155             echo "change yangtools version to ${YANGTOOLS_VERSION}"
 156             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
 157             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}"
 158         fi
 159         if [ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]; then
 160             MDSAL_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:mdsal.version ../pom.xml)"
 161             echo "change mdsal version to ${MDSAL_VERSION}"
 162             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:parent/x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
 163             find . -name "pom.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}"
 164         fi
 165     else
 166         # If project with no patch is MRI, download release tag:
 167         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 168             PROJECT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:${PROJECT_SHORTNAME}.version ../pom.xml)"
 169             echo "2. checking out tag v${PROJECT_VERSION}"
 170             git checkout tags/v${PROJECT_VERSION}
 171         # Otherwise download distribution branch:
 172         else
 173             echo "2. checking out branch ${DISTRIBUTION_BRANCH_TO_BUILD}"
 174             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 175         fi
 176     fi
 177     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 178     PICK_SEGMENT="$(echo "${patch}" | cut -d\: -s -f 2-)"
 179     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 180     for pick in "${PICKS[@]}"
 181     do
 182         echo "3. cherry-picking ${pick}"
 183         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 184         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 185     done
 186     cd "${BUILD_DIR}"
 187 done
 188
 189 # Finally add distribution if there is no int/dist patch
 190 if [ "${distribution_status}" == "not_included" ]; then
 191     echo "adding integration/distribution"
 192     PROJECTS+=(distribution)
 193     # clone distribution and add it as a module in root pom
 194     git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
 195     cd distribution
 196     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 197     cd "${BUILD_DIR}"
 198 fi
 199
 200 # Second phase: build everything
 201
 202 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 203     # Set Fast build if project is not in BUILD_NORMAL and BUILD_FAST is true
 204     if [[ "${BUILD_NORMAL}" != *"${PROJECT_SHORTNAME}"* ]] && ${BUILD_FAST}; then
 205         fast_option="-Pq"
 206     else
 207         fast_option=""
 208     fi
 209     pushd "${PROJECT_SHORTNAME}"
 210     # Build project
 211     "$MVN" clean install \
 212     -e ${fast_option} \
 213     -Dstream="$DISTROSTREAM" \
 214     -Dgitid.skip=false \
 215     -Dmaven.gitcommitid.skip=false \
 216     --global-settings "$GLOBAL_SETTINGS_FILE" \
 217     --settings "$SETTINGS_FILE" \
 218     $MAVEN_OPTIONS
 219     # Since we've installed the artifacts, we can clean the build and save
 220     # disk space
 221     "$MVN" clean \
 222     -e ${fast_option} \
 223     -Dstream="$DISTROSTREAM" \
 224     -Dgitid.skip=false \
 225     -Dmaven.gitcommitid.skip=false \
 226     --global-settings "$GLOBAL_SETTINGS_FILE" \
 227     --settings "$SETTINGS_FILE" \
 228     $MAVEN_OPTIONS
 229     popd
 230 done
 231