jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 # TODO: 1) clean up inline todo's below :)
   4 # TODO: 2) Use just a topic branch to create a distribution.  see this email:
   5 #          https://lists.opendaylight.org/pipermail/discuss/2015-December/006040.html
   6 # TODO: 3) Bubble up CSIT jobs that are triggered by the multipatch job to a jenkins
   7 #          parameter.  the default can be distribution-test which calls all CSIT jobs
   8 #          but being able to easily override it to a smaller subset (or none) will be
   9 #          helpful
  10
  11 # create a fresh empty place to build this custom distribution
  12 BUILD_DIR=${WORKSPACE}/patch_tester
  13 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
  14 MAVEN_OPTIONS="$(echo --show-version \
  15     --batch-mode \
  16     -Djenkins \
  17     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  18     -Dmaven.repo.local=/tmp/r \
  19     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)"
  20
  21 rm -rf $BUILD_DIR
  22 mkdir -p $BUILD_DIR
  23 cd $BUILD_DIR || exit 1
  24
  25 # Set up git committer name and email, needed for commit creation when cherry-picking.
  26 export EMAIL="sandbox@jenkins.opendaylight.org"
  27 export GIT_COMMITTER_NAME="Multipatch Job"
  28
  29 # Extract a list of patches per project from an comment trigger. An example is:
  30 # Patch Set 1:
  31 #
  32 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  33 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  34     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  35         BUILD_FAST="true"
  36         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  37     else
  38         BUILD_FAST="false"
  39         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  40     fi
  41     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  42 fi
  43 if ${BUILD_FAST}; then
  44     fast_option="-Pq"
  45 else
  46     fast_option=""
  47 fi
  48 # check if topic exists, e.g. topic=binding-tlc-rpc
  49 if [[ "${PATCHES_TO_BUILD}" == *topic* ]]; then
  50     TOPIC="${PATCHES_TO_BUILD#*=}"
  51     echo "Create topic ${TOPIC} patch list"
  52     PATCHES_TO_BUILD=""
  53     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  54     for PROJECT in "${PROJECT_LIST[@]}"; do
  55         # get all patches number for a topic for a given project
  56         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} \
  57         | grep 'number:' | awk '{{ print $2 }}')" || true
  58         echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  59         # add project if it is the first with patches or it is not the first
  60         if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  61             PATCHES_TO_BUILD="${PROJECT}"
  62         elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
  63             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  64         fi
  65         # sort project patches
  66         if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  67             REF_LIST=()
  68             # create reference list with patch number-refspec
  69             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  70                 REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
  71                 | grep 'ref:' | awk '{{ print $2 }}')
  72                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  73             done
  74             # sort reference list by patch number
  75             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  76             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  77             # add refspec to patches to build list
  78             for PATCH in "${SORT_REF_LIST[@]}"; do
  79                 PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  80             done
  81         fi
  82     done
  83 fi
  84
  85 echo "Patches to build: ${PATCHES_TO_BUILD}"
  86 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
  87
  88 # First phase: clone the necessary repos and set the patches up
  89
  90 declare -a PROJECTS
  91
  92 # For each patch:
  93 # * Clone the project.
  94 # * Optionally, checkout a specific (typically unmerged) Gerrit patch. If none,
  95 #   default to Integration/Distribution branch via {branch} JJB param.
  96 # * Also optionally, cherry-pick series of patches on top of the checkout.
  97 # * Final option: perform a 'release' by removing "-SNAPSHOT" everywhere within the project.
  98 #
  99 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 100 # list of project[=checkout][:cherry-pick]* values.
 101 #
 102 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 103 #
 104 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 105 #
 106 # Checkout patches for both odlparent and yangtools:
 107 #
 108 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 109 #
 110 # Checkout a patch for controller, cherry-pick another patch on top of it:
 111 #
 112 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 113 distribution_status="not_included"
 114 for proto_patch in "${PATCHES[@]}"
 115 do
 116     echo "working on ${proto_patch}"
 117     # For patch=controller=61/29761/5:45/29645/6, this gives controller
 118     PROJECT="$(echo ${proto_patch} | cut -d\: -f 1 | cut -d\= -f 1)"
 119     if [ "${PROJECT}" == "integration/distribution" ]; then
 120         distribution_status="included"
 121     fi
 122     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 123     PROJECTS+=("${PROJECT_SHORTNAME}")
 124     echo "cloning project ${PROJECT}"
 125     git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
 126     cd ${PROJECT_SHORTNAME} || exit 1
 127     if [ "$(echo -n ${proto_patch} | tail -c 1)" == 'r' ]; then
 128         pure_patch="$(echo -n $proto_patch | head -c -1)"
 129     else
 130         pure_patch="$proto_patch"
 131     fi
 132     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5
 133     CHECKOUT="$(echo ${pure_patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
 134     if [ "x${CHECKOUT}" != "x" ]; then
 135         echo "checking out ${CHECKOUT}"
 136         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 137         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 138         git checkout FETCH_HEAD
 139     else
 140         echo "checking out ${DISTRIBUTION_BRANCH_TO_BUILD}"
 141         git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 142     fi
 143     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 144     PICK_SEGMENT="$(echo "${pure_patch}" | cut -d\: -s -f 2-)"
 145     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 146     for pick in "${PICKS[@]}"
 147     do
 148         echo "cherry-picking ${pick}"
 149         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 150         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 151     done
 152     if [ "$(echo -n ${proto_patch} | tail -c 1)" == 'r' ]; then
 153         # Here 'r' means release. Useful for testing Nitrogen Odlparent changes.
 154         find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
 155     fi
 156     cd "${BUILD_DIR}" || exit 1
 157 done
 158
 159 if [ "${distribution_status}" == "not_included" ]; then
 160     echo "adding integration/distribution"
 161     PROJECTS+=(distribution)
 162     # clone distribution and add it as a module in root pom
 163     git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
 164     cd distribution || exit 1
 165     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 166     cd "${BUILD_DIR}" || exit 1
 167 fi
 168
 169 # Second phase: build everything
 170
 171 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 172     pushd "${PROJECT_SHORTNAME}" || exit 1
 173     # Build project
 174     "$MVN" clean install \
 175     -e ${fast_option} \
 176     -Dstream="$DISTROSTREAM" \
 177     -Dgitid.skip=false \
 178     -Dmaven.gitcommitid.skip=false \
 179     --global-settings "$GLOBAL_SETTINGS_FILE" \
 180     --settings "$SETTINGS_FILE" \
 181     $MAVEN_OPTIONS
 182     # Since we've installed the artifacts, we can clean the build and save
 183     # disk space
 184     "$MVN" clean \
 185     -e ${fast_option} \
 186     -Dstream="$DISTROSTREAM" \
 187     -Dgitid.skip=false \
 188     -Dmaven.gitcommitid.skip=false \
 189     --global-settings "$GLOBAL_SETTINGS_FILE" \
 190     --settings "$SETTINGS_FILE" \
 191     $MAVEN_OPTIONS
 192     popd || exit 1
 193 done
 194