jjb/integration/multipatch-distribution.sh

   1 #!/bin/bash
   2
   3 set -e
   4
   5 # create a fresh empty place to build this custom distribution
   6 BUILD_DIR=${WORKSPACE}/patch_tester
   7 DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH  #renaming variable for clarity
   8 MAVEN_OPTIONS="${MAVEN_PARAMS} \
   9     --show-version \
  10     --batch-mode \
  11     -Djenkins \
  12     -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
  13     -Dmaven.repo.local=/tmp/r \
  14     -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r"
  15
  16 rm -rf $BUILD_DIR
  17 mkdir -p $BUILD_DIR
  18 cd $BUILD_DIR
  19
  20 # Set up git committer name and email, needed for commit creation when cherry-picking.
  21 export EMAIL="sandbox@jenkins.opendaylight.org"
  22 export GIT_COMMITTER_NAME="Multipatch Job"
  23
  24 # Extract a list of patches per project from an comment trigger. An example is:
  25 # multipatch-build:openflowplugin:45/69445/1,genius:46/69446/1,netvirt:47/69447/1
  26 if [ -n "$GERRIT_EVENT_COMMENT_TEXT" ]; then
  27     if [[ "$GERRIT_EVENT_COMMENT_TEXT" == *fast* ]]; then
  28         BUILD_FAST="true"
  29         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build-fast:')
  30     else
  31         BUILD_FAST="false"
  32         PATCHES_TO_BUILD=$(echo "$GERRIT_EVENT_COMMENT_TEXT" | grep 'multipatch-build:')
  33     fi
  34     PATCHES_TO_BUILD=${PATCHES_TO_BUILD#*:}
  35 fi
  36 if ${BUILD_FAST}; then
  37     fast_option="-Pq"
  38 else
  39     fast_option=""
  40 fi
  41 # check if topic exists, e.g. topic=binding-tlc-rpc
  42 if [[ "${PATCHES_TO_BUILD}" == *topic* ]]; then
  43     TOPIC="${PATCHES_TO_BUILD#*=}"
  44     echo "Create topic ${TOPIC} patch list"
  45     PATCHES_TO_BUILD=""
  46     read -ra PROJECT_LIST <<< "${BUILD_ORDER}"
  47     echo "List of projects to check patch in topic: ${PROJECT_LIST[*]}"
  48     for PROJECT in "${PROJECT_LIST[@]}"; do
  49         # get all patches number for a topic for a given project
  50         IFS=$'\n' read -rd '' -a GERRIT_PATCH_LIST <<< "$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query status:open topic:${TOPIC} project:${PROJECT} \
  51         | grep 'number:' | awk '{{ print $2 }}')" || true
  52         # add project if it is the first with patches or it is not the first
  53         if [[ -z "${PATCHES_TO_BUILD}" && ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  54             PATCHES_TO_BUILD="${PROJECT}"
  55         elif [[ ! -z "${PATCHES_TO_BUILD}" ]]; then
  56             if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  57                 echo "Add ${PROJECT}:${DISTRIBUTION_BRANCH_TO_BUILD}"
  58             fi
  59             PATCHES_TO_BUILD="${PATCHES_TO_BUILD},${PROJECT}"
  60         fi
  61         # sort project patches
  62         if [[ ! -z "${GERRIT_PATCH_LIST[*]}" ]]; then
  63             echo "Add ${PROJECT}:${GERRIT_PATCH_LIST[*]}"
  64             REF_LIST=()
  65             # create reference list with patch number-refspec
  66             for PATCH in "${GERRIT_PATCH_LIST[@]}"; do
  67                 REFSPEC=$(ssh -p 29418 jenkins-$SILO@git.opendaylight.org gerrit query change:${PATCH} --current-patch-set \
  68                 | grep 'ref:' | awk '{{ print $2 }}')
  69                 REF_LIST+=("${PATCH}-${REFSPEC/refs\/changes\/}")
  70             done
  71             # sort reference list by patch number
  72             IFS=$'\n' SORT_REF=$(sort <<<"${REF_LIST[*]}") && unset IFS
  73             read -rd '' -a SORT_REF_LIST <<< "${SORT_REF[*]}" || true
  74             # add refspec to patches to build list
  75             for PATCH in "${SORT_REF_LIST[@]}"; do
  76                 # if project is odlparent or yangtools, do not cherry-pick
  77                 if [[ "${PROJECT}" == "odlparent" || "${PROJECT}" == "yangtools" ]]; then
  78                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}=${PATCH/*-/}"
  79                 else
  80                     PATCHES_TO_BUILD="${PATCHES_TO_BUILD}:${PATCH/*-/}"
  81                 fi
  82             done
  83         fi
  84     done
  85 fi
  86 echo "Patches to build: ${PATCHES_TO_BUILD}"
  87 IFS=',' read -ra PATCHES <<< "${PATCHES_TO_BUILD}"
  88
  89 # First phase: clone the necessary repos and set the patches up
  90
  91 declare -a PROJECTS
  92
  93 # For each patch:
  94 # 1. Clone the project.
  95 # 2. Optionally, checkout a specific (typically unmerged) Gerrit patch. If none,
  96 #   default to Integration/Distribution branch via {branch} JJB param.
  97 # 3. Also optionally, cherry-pick series of patches on top of the checkout.
  98 #
  99 # Each patch is found in the ${PATCHES_TO_BUILD} variable as a comma separated
 100 # list of project[=checkout][:cherry-pick]* values. Examples:
 101 #
 102 # Checkout a (typically unmerged) Gerrit patch on top of odlparent's git clone:
 103 # PATCHES_TO_BUILD='odlparent=45/30045/2'
 104 #
 105 # Checkout patches for both odlparent and yangtools:
 106 # PATCHES_TO_BUILD='odlparent=45/30045/2,yangtools:53/26853/25'
 107 #
 108 # Checkout a patch for controller, cherry-pick another patch on top of it:
 109 # PATCHES_TO_BUILD='controller=61/29761/5:45/29645/6'
 110 distribution_status="not_included"
 111 for patch in "${PATCHES[@]}"
 112 do
 113     echo "working on ${patch}"
 114     # For patch=controller=61/29761/5:45/29645/6, this gives controller
 115     PROJECT="$(echo ${patch} | cut -d\: -f 1 | cut -d\= -f 1)"
 116     if [ "${PROJECT}" == "integration/distribution" ]; then
 117         distribution_status="included"
 118     fi
 119     PROJECT_SHORTNAME="${PROJECT##*/}"  # http://stackoverflow.com/a/3162500
 120     PROJECTS+=("${PROJECT_SHORTNAME}")
 121     echo "cloning project ${PROJECT}"
 122     git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}"
 123     cd ${PROJECT_SHORTNAME}
 124     # For patch = controller=61/29761/5:45/29645/6, this gives 61/29761/5
 125     CHECKOUT="$(echo ${patch} | cut -d\= -s -f 2 | cut -d\: -f 1)"
 126     # If project has a patch, checkout patch, otherwise use distribution branch
 127     if [ "x${CHECKOUT}" != "x" ]; then
 128         echo "checking out ${CHECKOUT}"
 129         # TODO: Make this script accept "29645/6" as a shorthand for "45/29645/6".
 130         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/$CHECKOUT"
 131         git checkout FETCH_HEAD
 132
 133     else
 134         # If project with no patch = yangtools, download master branch
 135         if [ "${PROJECT}" == "yangtools" ]; then
 136             echo "checking out master"
 137             git checkout master
 138         else
 139             echo "checking out ${DISTRIBUTION_BRANCH_TO_BUILD}"
 140             git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 141         fi
 142     fi
 143     # For patch=controller=61/29761/5:45/29645/6, this gives 45/29645/6
 144     PICK_SEGMENT="$(echo "${patch}" | cut -d\: -s -f 2-)"
 145     IFS=':' read -ra PICKS <<< "${PICK_SEGMENT}"
 146     for pick in "${PICKS[@]}"
 147     do
 148         echo "cherry-picking ${pick}"
 149         git fetch "https://git.opendaylight.org/gerrit/${PROJECT}" "refs/changes/${pick}"
 150         git cherry-pick --ff --keep-redundant-commits FETCH_HEAD
 151     done
 152     cd "${BUILD_DIR}"
 153 done
 154
 155 # Finally add distribution if there is no int/dist patch
 156 if [ "${distribution_status}" == "not_included" ]; then
 157     echo "adding integration/distribution"
 158     PROJECTS+=(distribution)
 159     # clone distribution and add it as a module in root pom
 160     git clone "https://git.opendaylight.org/gerrit/p/integration/distribution"
 161     cd distribution
 162     git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}"
 163     cd "${BUILD_DIR}"
 164 fi
 165
 166 # If there is a patch for odlparent or yangtools (MRI projects), adjust version to mdsal project:
 167 # 1. Extract project version in patch
 168 # 2. Extract project MSI version from mdsal project
 169 # 3. Replace version in patch by MSI version
 170 # Otherwise release the MRI project
 171
 172 if [[ -d "odlparent" ]]; then
 173     if [[ -d "mdsal" ]]; then
 174         # Extract patch and MSI used odlparent version
 175         patch_version="$(xpath ./odlparent/odlparent-lite/pom.xml '/project/version/text()' 2> /dev/null)"
 176         msi_version="$(xpath ./mdsal/pom.xml '/project/parent/version/text()' 2> /dev/null)"
 177         # Replace odlparent version
 178         find ./odlparent -name "*.xml" -print0 | xargs -0 sed -i "s/${patch_version}/${msi_version}/g"
 179         echo "odlparent project version changed to ${msi_version}"
 180     else
 181         # Release odlparent
 182         find ./odlparent -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
 183         odlparent_version=${patch_version%"-SNAPSHOT"}
 184         echo "odlparent project version changed to ${odlparent_version}"
 185     fi
 186 fi
 187 if [[ -d "yangtools" ]]; then
 188     if [[ -d "mdsal" ]]; then
 189         # Adjust yangtools and odlparent version to mdsal
 190         # Extract patch and MSI used yangtools version
 191         patch_version="$(xpath ./yangtools/pom.xml '/project/version/text()' 2> /dev/null)"
 192         msi_version="$(xpath ./mdsal/dom/dom-parent/pom.xml '/project/dependencyManagement/dependencies/dependency[artifactId="yangtools-artifacts"]/version/text()' 2> /dev/null)"
 193         # Replace yangtools version
 194         find ./yangtools -name "*.xml" -print0 | xargs -0 sed -i "s/${patch_version}/${msi_version}/g"
 195         echo "yangtools project version changed to ${msi_version}"
 196         # Extract patch and MSI used odlparent version
 197         patch_version="$(xpath ./yangtools/pom.xml '/project/parent/version/text()' 2> /dev/null)"
 198         msi_version="$(xpath ./mdsal/pom.xml '/project/parent/version/text()' 2> /dev/null)"
 199         # Replace odlparent version
 200         find ./yangtools -name "*.xml" -print0 | xargs -0 sed -i "s/${patch_version}/${msi_version}/g"
 201         echo "yangtools project odlparent version changed to ${msi_version}"
 202     else
 203         # Release yangtools and adjust odlparent version if required
 204         find ./yangtools -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g'
 205         yangtools_version=${patch_version%"-SddNAPSHOT"}
 206         echo "yangtools project version changed to ${yangtools_version}"
 207         if [[ -d "odlparent" ]]; then
 208             # Extract odlparent version from odlparent
 209             patch_version="$(xpath ./yangtools/pom.xml '/project/parent/version/text()' 2> /dev/null)"
 210             # Replace odlparent version
 211             find ./yangtools -name "*.xml" -print0 | xargs -0 sed -i "s/${patch_version}/${odlparent_version}/g"
 212             echo "yangtools project odlparent version changed to ${odlparent_version}"
 213         fi
 214     fi
 215 fi
 216
 217 # Second phase: build everything
 218
 219 for PROJECT_SHORTNAME in "${PROJECTS[@]}"; do
 220     pushd "${PROJECT_SHORTNAME}"
 221     # Build project
 222     "$MVN" clean install \
 223     -e ${fast_option} \
 224     -Dstream="$DISTROSTREAM" \
 225     -Dgitid.skip=false \
 226     -Dmaven.gitcommitid.skip=false \
 227     --global-settings "$GLOBAL_SETTINGS_FILE" \
 228     --settings "$SETTINGS_FILE" \
 229     $MAVEN_OPTIONS
 230     # Since we've installed the artifacts, we can clean the build and save
 231     # disk space
 232     "$MVN" clean \
 233     -e ${fast_option} \
 234     -Dstream="$DISTROSTREAM" \
 235     -Dgitid.skip=false \
 236     -Dmaven.gitcommitid.skip=false \
 237     --global-settings "$GLOBAL_SETTINGS_FILE" \
 238     --settings "$SETTINGS_FILE" \
 239     $MAVEN_OPTIONS
 240     popd
 241 done
 242