jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -e
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="$(echo --show-version \
   9     --batch-mode \
  10     -Djenkins \
  11     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  12     -Dmaven.repo.local=/tmp/r \
  13     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)"
  14
  15 rm -rf $BUILD_DIR
  16 mkdir -p $BUILD_DIR
  17 cd $BUILD_DIR
  18
  19 # Set up git committer name and email, needed for commit creation when cherry-picking.
  20 export EMAIL="sandbox@jenkins.opendaylight.org"
  21 export GIT_COMMITTER_NAME="Multipatch Job"
  22
  23 # Extract a list of patches per project from an comment trigger. An example is:
  24 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  25 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  26     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  27         BUILD_FAST="true"
  28         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  29     else
  30         BUILD_FAST="false"
  31         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  32     fi
  33     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  34 fi
  35 if ${BUILD_FAST}; then
  36     fast_option="-Pq"
  37 else
  38     fast_option=""
  39 fi
  40 # check if topic exists, e.g. topic=binding-tlc-rpc
  41 if [[ "${PATCHES_TO_BUILD}" == *topic* ]]; then
  42     TOPIC="${PATCHES_TO_BUILD#*=}"
  43     echo "Create topic ${TOPIC} patch list"
  44     PATCHES_TO_BUILD=""
  45     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  46     for PROJECT in "${PROJECT_LIST[@]}"; do
  47         # get all patches number for a topic for a given project
  48         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} \
  49         | grep 'number:' | awk '{{ print $2 }}')" || true
  50         # add project if it is the first with patches or it is not the first
  51         if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  52             PATCHES_TO_BUILD="${PROJECT}"
  53         elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
  54             if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  55                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  56             fi
  57             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  58         fi
  59         # sort project patches
  60         if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  61             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  62             REF_LIST=()
  63             # create reference list with patch number-refspec
  64             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  65                 REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
  66                 | grep 'ref:' | awk '{{ print $2 }}')
  67                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  68             done
  69             # sort reference list by patch number
  70             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  71             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  72             # add refspec to patches to build list
  73             for PATCH in "${SORT_REF_LIST[@]}"; do
  74                 PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  75             done
  76         fi
  77     done
  78 fi
  79
  80 echo "Patches to build: ${PATCHES_TO_BUILD}"
  81 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
  82
  83 # First phase: clone the necessary repos and set the patches up
  84
  85 declare -a PROJECTS
  86
  87 # For each patch:
  88 # 1. Clone the project.
  89 # 2. Optionally, checkout a specific (typically unmerged) Gerrit patch. If none,
  90 #   default to Integration/Distribution branch via {branch} JJB param.
  91 # 3. Also optionally, cherry-pick series of patches on top of the checkout.
  92 #
  93 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
  94 # list of project[=checkout][:cherry-pick]* values. Examples:
  95 #
  96 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
  97 # PATCHES_TO_BUILD='odlparent=45/30045/2'
  98 #
  99 # Checkout patches for both odlparent and yangtools:
 100 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 101 #
 102 # Checkout a patch for controller, cherry-pick another patch on top of it:
 103 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 104 distribution_status="not_included"
 105 for patch in "${PATCHES[@]}"
 106 do
 107     echo "working on ${patch}"
 108     # For patch=controller=61/29761/5:45/29645/6, this gives controller
 109     PROJECT="$(echo ${patch} | cut -d\: -f 1 | cut -d\= -f 1)"
 110     if [ "${PROJECT}" == "integration/distribution" ]; then
 111         distribution_status="included"
 112     fi
 113     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 114     PROJECTS+=("${PROJECT_SHORTNAME}")
 115     echo "cloning project ${PROJECT}"
 116     git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
 117     cd ${PROJECT_SHORTNAME}
 118     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5
 119     CHECKOUT="$(echo ${patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
 120     # If project has a patch, checkout patch, otherwise use distribution branch
 121     if [ "x${CHECKOUT}" != "x" ]; then
 122         echo "checking out ${CHECKOUT}"
 123         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 124         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 125         git checkout FETCH_HEAD
 126
 127     else
 128         # If project with no patch = yangtools, download master branch
 129         if [ "${PROJECT}" == "yangtools" ]; then
 130             echo "checking out master"
 131             git checkout master
 132         else
 133             echo "checking out ${DISTRIBUTION_BRANCH_TO_BUILD}"
 134             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 135         fi
 136     fi
 137     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 138     PICK_SEGMENT="$(echo "${patch}" | cut -d\: -s -f 2-)"
 139     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 140     for pick in "${PICKS[@]}"
 141     do
 142         echo "cherry-picking ${pick}"
 143         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 144         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 145     done
 146     cd "${BUILD_DIR}"
 147 done
 148
 149 # Finally add distribution if there is no int/dist patch
 150 if [ "${distribution_status}" == "not_included" ]; then
 151     echo "adding integration/distribution"
 152     PROJECTS+=(distribution)
 153     # clone distribution and add it as a module in root pom
 154     git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
 155     cd distribution
 156     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 157     cd "${BUILD_DIR}"
 158 fi
 159
 160 # If there is a patch for odlparent or yangtools (MRI projects), adjust version to mdsal project:
 161 # 1. Extract project version in patch
 162 # 2. Extract project MSI version from mdsal project
 163 # 3. Replace version in patch by MSI version
 164 # Otherwise release the MRI project
 165 if [[ -d "odlparent" ]]; then
 166     if [[ -d "mdsal" ]]; then
 167         # Extract patch and MSI used version
 168         patch_version="$(xpath ./odlparent/odlparent-lite/pom.xml '/project/version/text()' 2> /dev/null)"
 169         msi_version="$(xpath ./mdsal/pom.xml '/project/parent/version/text()' 2> /dev/null)"
 170         # Replace version
 171         find ./odlparent -name "*.xml" -print0 | xargs -0 sed -i "s/${patch_version}/${msi_version}/g"
 172     else
 173         # Release project
 174         find ./odlparent -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
 175     fi
 176 fi
 177 if [[ -d "yangtools" ]]; then
 178         if [[ -d "mdsal" ]]; then
 179         # Extract patch and MSI used version
 180         patch_version="$(xpath ./yangtools/pom.xml '/project/version/text()' 2> /dev/null)"
 181         msi_version="$(xpath ./mdsal/binding/yang-binding/pom.xml '/project/dependencyManagement/dependencies/dependency/version/text()' 2> /dev/null)"
 182         # Replace version
 183         find ./yangtools -name "*.xml" -print0 | xargs -0 sed -i "s/${patch_version}/${msi_version}/g"
 184     else
 185         # Release project
 186         find ./yangtools -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
 187     fi
 188 fi
 189
 190 # Second phase: build everything
 191
 192 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 193     pushd "${PROJECT_SHORTNAME}"
 194     # Build project
 195     "$MVN" clean install \
 196     -e ${fast_option} \
 197     -Dstream="$DISTROSTREAM" \
 198     -Dgitid.skip=false \
 199     -Dmaven.gitcommitid.skip=false \
 200     --global-settings "$GLOBAL_SETTINGS_FILE" \
 201     --settings "$SETTINGS_FILE" \
 202     $MAVEN_OPTIONS
 203     # Since we've installed the artifacts, we can clean the build and save
 204     # disk space
 205     "$MVN" clean \
 206     -e ${fast_option} \
 207     -Dstream="$DISTROSTREAM" \
 208     -Dgitid.skip=false \
 209     -Dmaven.gitcommitid.skip=false \
 210     --global-settings "$GLOBAL_SETTINGS_FILE" \
 211     --settings "$SETTINGS_FILE" \
 212     $MAVEN_OPTIONS
 213     popd
 214 done
 215