jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -e
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="${MAVEN_PARAMS} \
   9     --show-version \
  10     --batch-mode \
  11     -Djenkins \
  12     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  13     -Dmaven.repo.local=/tmp/r \
  14     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
  15
  16 rm -rf $BUILD_DIR
  17 mkdir -p $BUILD_DIR
  18 cd $BUILD_DIR
  19
  20 # Download distribution pom.xml
  21 wget "http://git.opendaylight.org/gerrit/gitweb?p=integration/distribution.git;a=blob_plain;f=artifacts/upstream/properties/pom.xml;hb=refs/heads/$DISTROBRANCH" -O "pom.xml"
  22 cat pom.xml
  23
  24 # Set up git committer name and email, needed for commit creation when cherry-picking.
  25 export EMAIL="sandbox@jenkins.opendaylight.org"
  26 export GIT_COMMITTER_NAME="Multipatch Job"
  27
  28 # Extract a list of patches per project from an comment trigger. An example is:
  29 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  30 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  31     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  32         BUILD_FAST="true"
  33         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  34     else
  35         BUILD_FAST="false"
  36         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  37     fi
  38     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  39 fi
  40 if ${BUILD_FAST}; then
  41     fast_option="-Pq"
  42 else
  43     fast_option=""
  44 fi
  45 # check if topic exists, e.g. topic=binding-tlc-rpc
  46 if [[ "${PATCHES_TO_BUILD}" == *topic* ]]; then
  47     TOPIC="${PATCHES_TO_BUILD#*=}"
  48     echo "Create topic ${TOPIC} patch list"
  49     PATCHES_TO_BUILD=""
  50     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  51     echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
  52     for PROJECT in "${PROJECT_LIST[@]}"; do
  53         # get all patches number for a topic for a given project
  54         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} 2> /dev/null \
  55         | grep 'number:' | awk '{{ print $2 }}')" || true
  56         # add project if it is the first with patches or it is not the first
  57         if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  58             PATCHES_TO_BUILD="${PROJECT}"
  59         elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
  60             if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  61                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  62             fi
  63             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  64         fi
  65         # sort project patches
  66         if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  67             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  68             REF_LIST=()
  69             # create reference list with patch number-refspec
  70             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  71                 REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
  72                 | grep 'ref:' | awk '{{ print $2 }}')
  73                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  74             done
  75             # sort reference list by patch number
  76             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  77             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  78             # add refspec to patches to build list
  79             for PATCH in "${SORT_REF_LIST[@]}"; do
  80                 # cherry-pick is better than checkout patch
  81                 PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  82             done
  83         fi
  84     done
  85 fi
  86 echo "Patches to build: ${PATCHES_TO_BUILD}"
  87 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
  88
  89 # First phase: clone the necessary repos and set the patches up
  90
  91 declare -a PROJECTS
  92
  93 # For each patch:
  94 # 1. Clone the project.
  95 # 2. Checkout an specific (typically unmerged) Gerrit patch. If none,
  96 # use distribution pom.xml file to figure out right branch or tag to checkout.
  97 # In case of Gerrit patch in MRI project, adjust version for the stream.
  98 # 3. Optionally, cherry-pick series of patches on top of the checkout.
  99 #
 100 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 101 # list of project[=checkout][:cherry-pick]* values. Examples:
 102 #
 103 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 104 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 105 #
 106 # Checkout patches for both odlparent and yangtools:
 107 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 108 #
 109 # Checkout a patch for controller, cherry-pick another patch on top of it:
 110 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 111 distribution_status="not_included"
 112 for patch in "${PATCHES[@]}"
 113 do
 114     echo "-- working on ${patch} --"
 115     # For patch=controller=61/29761/5:45/29645/6, this gives controller.
 116     PROJECT="$(echo ${patch} | cut -d\: -f 1 | cut -d\= -f 1)"
 117     if [ "${PROJECT}" == "integration/distribution" ]; then
 118         distribution_status="included"
 119     fi
 120     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 121     PROJECTS+=("${PROJECT_SHORTNAME}")
 122     echo "1. cloning project ${PROJECT}"
 123     git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
 124     cd ${PROJECT_SHORTNAME}
 125     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5.
 126     CHECKOUT="$(echo ${patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
 127     # If there is a base patch for this project, checkout patch, otherwise use
 128     # distribution pom.xml file to figure out right branch or tag to checkout.
 129     if [ "x${CHECKOUT}" != "x" ]; then
 130         echo "2. checking out patch ${CHECKOUT}"
 131         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 132         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 133         git checkout FETCH_HEAD
 134         # If the patch is for MRI project, adjust the MRI versions
 135         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 136             ODLPARENT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:odlparent.version ../pom.xml)"
 137             echo "change odlparent version to ${ODLPARENT_VERSION}"
 138             find . -name "*.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.odlparent\"\] -v "${ODLPARENT_VERSION}" 2> /dev/null
 139         fi
 140         if [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 141             YANGTOOLS_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:yangtools.version ../pom.xml)"
 142             echo "change yangtools version to ${YANGTOOLS_VERSION}"
 143             find -name "*.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.yangtools\"\] -v "${YANGTOOLS_VERSION}" 2> /dev/null
 144         fi
 145         if [ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]; then
 146             MDSAL_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:mdsal.version ../pom.xml)"
 147             echo "change mdsal version to ${MDSAL_VERSION}"
 148             find -name "*.xml" -print0 | xargs -0 xmlstarlet ed --inplace -P -N x=http://maven.apache.org/POM/4.0.0 -u //x:version\[../x:groupId=\"org.opendaylight.mdsal\"\] -v "${MDSAL_VERSION}" 2> /dev/null
 149         fi
 150     else
 151         # If project with no patch is MRI, download release tag:
 152         if [ "${PROJECT}" == "odlparent" ] || [ "${PROJECT}" == "yangtools" ] || ([ "${PROJECT}" == "mdsal" ] && [ "${DISTROSTREAM}" != "fluorine" ]); then
 153             PROJECT_VERSION="$(xmlstarlet sel -N x=http://maven.apache.org/POM/4.0.0 -t -v //x:${PROJECT_SHORTNAME}.version ../pom.xml)"
 154             echo "2. checking out tag v${PROJECT_VERSION}"
 155             git checkout tags/v${PROJECT_VERSION}
 156         # Otherwise download distribution branch:
 157         else
 158             echo "2. checking out branch ${DISTRIBUTION_BRANCH_TO_BUILD}"
 159             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 160         fi
 161     fi
 162     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 163     PICK_SEGMENT="$(echo "${patch}" | cut -d\: -s -f 2-)"
 164     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 165     for pick in "${PICKS[@]}"
 166     do
 167         echo "3. cherry-picking ${pick}"
 168         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 169         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 170     done
 171     cd "${BUILD_DIR}"
 172 done
 173
 174 # Finally add distribution if there is no int/dist patch
 175 if [ "${distribution_status}" == "not_included" ]; then
 176     echo "adding integration/distribution"
 177     PROJECTS+=(distribution)
 178     # clone distribution and add it as a module in root pom
 179     git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
 180     cd distribution
 181     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 182     cd "${BUILD_DIR}"
 183 fi
 184
 185 # Second phase: build everything
 186
 187 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 188     pushd "${PROJECT_SHORTNAME}"
 189     # Build project
 190     "$MVN" clean install \
 191     -e ${fast_option} \
 192     -Dstream="$DISTROSTREAM" \
 193     -Dgitid.skip=false \
 194     -Dmaven.gitcommitid.skip=false \
 195     --global-settings "$GLOBAL_SETTINGS_FILE" \
 196     --settings "$SETTINGS_FILE" \
 197     $MAVEN_OPTIONS
 198     # Since we've installed the artifacts, we can clean the build and save
 199     # disk space
 200     "$MVN" clean \
 201     -e ${fast_option} \
 202     -Dstream="$DISTROSTREAM" \
 203     -Dgitid.skip=false \
 204     -Dmaven.gitcommitid.skip=false \
 205     --global-settings "$GLOBAL_SETTINGS_FILE" \
 206     --settings "$SETTINGS_FILE" \
 207     $MAVEN_OPTIONS
 208     popd
 209 done
 210