jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -e
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="${MAVEN_PARAMS} \
   9     --show-version \
  10     --batch-mode \
  11     -Djenkins \
  12     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  13     -Dmaven.repo.local=/tmp/r \
  14     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
  15
  16 rm -rf $BUILD_DIR
  17 mkdir -p $BUILD_DIR
  18 cd $BUILD_DIR
  19
  20 # Download distribution pom.xml
  21 wget "http://git.opendaylight.org/gerrit/gitweb?p=integration/distribution.git;a=blob_plain;f=artifacts/upstream/properties/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
  22 cat pom.xml
  23
  24 # Set up git committer name and email, needed for commit creation when cherry-picking.
  25 export EMAIL="sandbox@jenkins.opendaylight.org"
  26 export GIT_COMMITTER_NAME="Multipatch Job"
  27
  28 # Extract a list of patches per project from an comment trigger. An example is:
  29 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  30 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  31     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  32         BUILD_FAST="true"
  33         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  34     else
  35         BUILD_FAST="false"
  36         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  37     fi
  38     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  39 fi
  40 if ${BUILD_FAST}; then
  41     fast_option="-Pq"
  42 else
  43     fast_option=""
  44 fi
  45 # check if topic exists:
  46 # if topic=binding-rpc, then checkout first patch in binding-rpc topic (if it exists)
  47 # if topic:binding-rpc, then cherry-pick first patch in binding-rpc topic (if it exists)
  48 if [[ "${PATCHES_TO_BUILD}" == *"topic"* ]]; then
  49     if [[ "${PATCHES_TO_BUILD}" == *"topic="* ]]; then
  50         CHERRY_PICK="false"
  51         TOPIC="${PATCHES_TO_BUILD#*=}"
  52     elif [[ "${PATCHES_TO_BUILD}" == *"topic:"* ]]; then
  53         CHERRY_PICK="true"
  54         TOPIC="${PATCHES_TO_BUILD#*:}"
  55     else
  56         echo "ERROR: Topic has wrong format" && exit 1
  57     fi
  58     echo "Create topic ${TOPIC} patch list"
  59     PATCHES_TO_BUILD=""
  60     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  61     echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
  62     for PROJECT in "${PROJECT_LIST[@]}"; do
  63         # get all patches number for a topic for a given project
  64         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} 2> /dev/null \
  65         | grep 'number:' | awk '{{ print $2 }}')" || true
  66         # add project if it is the first with patches or it is not the first
  67         if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  68             PATCHES_TO_BUILD="${PROJECT}"
  69         elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
  70             if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  71                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  72             fi
  73             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  74         fi
  75         # sort project patches
  76         if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  77             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  78             REF_LIST=()
  79             # create reference list with patch number-refspec
  80             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  81                 REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
  82                 | grep 'ref:' | awk '{{ print $2 }}')
  83                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  84             done
  85             # sort reference list by patch number
  86             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  87             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  88             # add refspec to patches to build list
  89             COUNT=0
  90             for PATCH in "${SORT_REF_LIST[@]}"; do
  91                 COUNT=$((COUNT+1))
  92                 if [ "${COUNT}" == "1" ] && [ "${CHERRY_PICK}" == "false" ]; then
  93                     # checkout patch
  94                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}=${PATCH/*-/}"
  95                 else
  96                     # cherry-pick is better than checkout patch
  97                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  98                 fi
  99             done
 100         fi
 101     done
 102 fi
 103 echo "Patches to build: ${PATCHES_TO_BUILD}"
 104 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
 105
 106 # First phase: clone the necessary repos and set the patches up
 107
 108 declare -a PROJECTS
 109
 110 # For each patch:
 111 # 1. Clone the project.
 112 # 2. Checkout an specific (typically unmerged) Gerrit patch. If none,
 113 # use distribution pom.xml file to figure out right branch or tag to checkout.
 114 # In case of Gerrit patch in MRI project, adjust version for the stream.
 115 # 3. Optionally, cherry-pick series of patches on top of the checkout.
 116 #
 117 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 118 # list of project[=checkout][:cherry-pick]* values. Examples:
 119 #
 120 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 121 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 122 #
 123 # Checkout patches for both odlparent and yangtools:
 124 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 125 #
 126 # Checkout a patch for controller, cherry-pick another patch on top of it:
 127 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 128 distribution_status="not_included"
 129 for patch in "${PATCHES[@]}"
 130 do
 131     echo "-- working on ${patch} --"
 132     # For patch=controller=61/29761/5:45/29645/6, this gives controller.
 133     PROJECT="$(echo ${patch} | cut -d\: -f 1 | cut -d\= -f 1)"
 134     if [ "${PROJECT}" == "integration/distribution" ]; then
 135         distribution_status="included"
 136     fi
 137     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 138     PROJECTS+=("${PROJECT_SHORTNAME}")
 139     echo "1. cloning project ${PROJECT}"
 140     git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
 141     cd ${PROJECT_SHORTNAME}
 142     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5.
 143     CHECKOUT="$(echo ${patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
 144     # If there is a base patch for this project, checkout patch, otherwise use
 145     # distribution pom.xml file to figure out right branch or tag to checkout.
 146     if [ "x${CHECKOUT}" != "x" ]; then
 147         echo "2. checking out patch ${CHECKOUT}"
 148         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 149         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 150         git checkout FETCH_HEAD
 151         # If the patch is for MRI project, adjust the MRI versions
 152         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 153             ODLPARENT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:odlparent.version ../pom.xml)"
 154             echo "change odlparent version to ${ODLPARENT_VERSION}"
 155             find . -name "*.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}" 2> /dev/null
 156         fi
 157         if [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 158             YANGTOOLS_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:yangtools.version ../pom.xml)"
 159             echo "change yangtools version to ${YANGTOOLS_VERSION}"
 160             find -name "*.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}" 2> /dev/null
 161         fi
 162         if [ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]; then
 163             MDSAL_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:mdsal.version ../pom.xml)"
 164             echo "change mdsal version to ${MDSAL_VERSION}"
 165             find -name "*.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}" 2> /dev/null
 166         fi
 167     else
 168         # If project with no patch is MRI, download release tag:
 169         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 170             PROJECT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:${PROJECT_SHORTNAME}.version ../pom.xml)"
 171             echo "2. checking out tag v${PROJECT_VERSION}"
 172             git checkout tags/v${PROJECT_VERSION}
 173         # Otherwise download distribution branch:
 174         else
 175             echo "2. checking out branch ${DISTRIBUTION_BRANCH_TO_BUILD}"
 176             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 177         fi
 178     fi
 179     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 180     PICK_SEGMENT="$(echo "${patch}" | cut -d\: -s -f 2-)"
 181     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 182     for pick in "${PICKS[@]}"
 183     do
 184         echo "3. cherry-picking ${pick}"
 185         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 186         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 187     done
 188     cd "${BUILD_DIR}"
 189 done
 190
 191 # Finally add distribution if there is no int/dist patch
 192 if [ "${distribution_status}" == "not_included" ]; then
 193     echo "adding integration/distribution"
 194     PROJECTS+=(distribution)
 195     # clone distribution and add it as a module in root pom
 196     git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
 197     cd distribution
 198     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 199     cd "${BUILD_DIR}"
 200 fi
 201
 202 # Second phase: build everything
 203
 204 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 205     pushd "${PROJECT_SHORTNAME}"
 206     # Build project
 207     "$MVN" clean install \
 208     -e ${fast_option} \
 209     -Dstream="$DISTROSTREAM" \
 210     -Dgitid.skip=false \
 211     -Dmaven.gitcommitid.skip=false \
 212     --global-settings "$GLOBAL_SETTINGS_FILE" \
 213     --settings "$SETTINGS_FILE" \
 214     $MAVEN_OPTIONS
 215     # Since we've installed the artifacts, we can clean the build and save
 216     # disk space
 217     "$MVN" clean \
 218     -e ${fast_option} \
 219     -Dstream="$DISTROSTREAM" \
 220     -Dgitid.skip=false \
 221     -Dmaven.gitcommitid.skip=false \
 222     --global-settings "$GLOBAL_SETTINGS_FILE" \
 223     --settings "$SETTINGS_FILE" \
 224     $MAVEN_OPTIONS
 225     popd
 226 done
 227