From 5498e03244f45d200a61553489a82309c54eaedc Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Thu, 18 Jan 2018 02:05:37 -0800 Subject: [PATCH] Fix multipatch job for upstream bump verification - Main change is to do ordered per-project build rather than a single build with a crafted root pom file. Change-Id: Ifac781e379c192aa32601c57fd833936b0b82c05 Signed-off-by: Luis Gomez --- jjb/integration/integration-test-jobs.yaml | 54 ++++++---------------- jjb/integration/multipatch-distribution.sh | 37 +++++++++------ jjb/releng-macros.yaml | 3 +- 3 files changed, 37 insertions(+), 57 deletions(-) diff --git a/jjb/integration/integration-test-jobs.yaml b/jjb/integration/integration-test-jobs.yaml index 5b7637193..9317ee400 100644 --- a/jjb/integration/integration-test-jobs.yaml +++ b/jjb/integration/integration-test-jobs.yaml @@ -25,6 +25,7 @@ csit-sanity-list: '{csit-sanity-list-oxygen}' csit-weekly-list: '{csit-weekly-list-oxygen}' csit-high-frequency-list: '{csit-high-frequency-list-oxygen}' + mvn-version: 'mvn33' - nitrogen: branch: 'stable/nitrogen' @@ -33,6 +34,7 @@ csit-sanity-list: '{csit-sanity-list-nitrogen}' csit-weekly-list: '{csit-weekly-list-nitrogen}' csit-high-frequency-list: '{csit-high-frequency-list-nitrogen}' + mvn-version: 'mvn33' - carbon: branch: 'stable/carbon' @@ -42,6 +44,7 @@ csit-sanity-list: '{csit-sanity-list-carbon}' csit-weekly-list: '{csit-weekly-list-carbon}' csit-high-frequency-list: '{csit-high-frequency-list-carbon}' + mvn-version: 'mvn33' # tools system image tools_system_image: ZZCI - Ubuntu 16.04 - mininet-ovs-25 - 20171208-1847 @@ -227,30 +230,12 @@ - job-template: name: 'integration-multipatch-test-{stream}' # Goal: Build one or more patches and run all available system test on a distribution containing the change. - # Operation: This job template checks out and builds each patch provided in job parameters, creates a distribution - # containing these patches, and triggers the distribution test job which includes most (if not all) available - # CSIT jobs - # TODO: Move the following to appropriate readthedocs document. - # Recommended Sandbox usage: - # 0. Create this job and a CSIT job you want to run on the new .zip in Sandbox (disable timed triggers). - # 1. Copy the csit job name to clipboard. - # 2. Navigate to Sandbox page of this job. - # 3. Click "Configure". - # 4. Locate "Trigger/call builds on other projects" (under "Post Steps" tab). - # 5. Delete the long value of "Projects to build" field and paste the job name from clipboard (ctrl+a, ctrl+v). - # 6. Click "Save" button. - # 7. Click "Build with Parameters". - # yamllint disable-line rule:line-length - # 8. Start the job with your parameters, see https://wiki.opendaylight.org/view/Integration/Test/Running_System_Tests#Running_System_Tests_Using_Custom_Distribution_Built_From_Multiple_Patches - # 9. The multipatch job waits for the csit job to finish, so abort csit if you see something went wrong. - # 10. Navigate to Sandbox page of the csit job, click the run triggered from multipatch. - # 11. Click Parameters and copy the long value of BUNDLE_URL field to clipboard. - # 12. Run the same csit job woth different patchset or test options, or other csit jobs, with this BUNDLE_URL value - # so you do not have to wait for multipatch build. Repeat step 12. - # 13. Currently, the .zip lasts 24 hours (?) so you may need to jump to step 2 (or 0) after some time. + # Operation: This job template checks out and builds in order each patch provided in job parameters, finally + # it creates a distribution containing all patches and triggers the distribution test job which includes most + # (if not all) available CSIT jobs. project-type: freestyle - node: centos7-builder-2c-8g + node: centos7-builder-4c-4g jdk: '{jdk}' properties: @@ -264,6 +249,8 @@ run-test: 'false' - integration-distribution-branch: branch: '{branch}' + - integration-distribution-stream: + stream: '{stream}' - distribution-karaf-version: karaf-version: '{karaf-version}' - opendaylight-infra-parameters: @@ -294,27 +281,12 @@ builders: - wipe-org-opendaylight-repo - - integration-multipatch-builder + - lf-maven-install: + mvn-version: '{mvn-version}' - provide-maven-settings: global-settings-file: 'global-settings' - settings-file: 'yangtools-settings' - - maven-target: - maven-version: mvn33 - pom: patch_tester/pom.xml - # TODO: Make Maven options configurable. Currently tests are not skipped, everything else is. - goals: | - clean install dependency:tree -DoutputFile=dependency_tree.txt - -Pq - -Dstream={stream} - -Dsft.heap.max=4g - {opendaylight-infra-mvn-opts} - # TODO: {opendaylight-infra-parallel-mvn-opts}? - java-opts: - - '-Xmx3072m -Dmaven.compile.fork=true' - settings: integration-settings - settings-type: cfp - global-settings: global-settings - global-settings-type: cfp + settings-file: 'integration-distribution-settings' + - integration-multipatch-builder - integration-upload-distribution: dist-pom: patch_tester/distribution/pom.xml - conditional-step: diff --git a/jjb/integration/multipatch-distribution.sh b/jjb/integration/multipatch-distribution.sh index f5711aedd..16cc153ad 100644 --- a/jjb/integration/multipatch-distribution.sh +++ b/jjb/integration/multipatch-distribution.sh @@ -8,22 +8,18 @@ # create a fresh empty place to build this custom distribution BUILD_DIR=${WORKSPACE}/patch_tester -POM_FILE=${WORKSPACE}/patch_tester/pom.xml DISTRIBUTION_BRANCH_TO_BUILD=$DISTROBRANCH #renaming variable for clarity +MAVEN_OPTIONS="$(echo --show-version \ + --batch-mode \ + -Djenkins \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dmaven.repo.local=/tmp/r \ + -Dorg.ops4j.pax.url.mvn.localRepository=/tmp/r)" rm -rf $BUILD_DIR mkdir -p $BUILD_DIR cd $BUILD_DIR || exit 1 -# create a root pom that will contain a module for each project we have a patch for -echo "" >> $POM_FILE -echo "org.opendaylight.test" >> $POM_FILE -echo "test" >> $POM_FILE -echo "0.1" >> $POM_FILE -echo "4.0.0" >> $POM_FILE -echo "pom" >> $POM_FILE -echo "" >> $POM_FILE - # Set up git committer name and email, needed for commit creation when cherry-picking. export EMAIL="sandbox@jenkins.opendaylight.org" # TODO: Is there a more appropriate e-mail? @@ -69,7 +65,6 @@ do PROJECT_SHORTNAME="${PROJECT##*/}" # http://stackoverflow.com/a/3162500 echo "cloning project ${PROJECT}" git clone "https://git.opendaylight.org/gerrit/p/${PROJECT}" - echo "${PROJECT_SHORTNAME}" >> ${POM_FILE} cd ${PROJECT_SHORTNAME} || exit 1 if [ "$(echo -n ${proto_patch} | tail -c 1)" == 'r' ]; then pure_patch="$(echo -n $proto_patch | head -c -1)" @@ -100,6 +95,14 @@ do # Here 'r' means release. Useful for testing Nitrogen Odlparent changes. find . -name "*.xml" -print0 | xargs -0 sed -i 's/-SNAPSHOT//g' fi + # Build project + "$MVN" clean install \ + -e -Pq \ + -Dstream=oxygen \ + -DskipTests=true \ + --global-settings "$GLOBAL_SETTINGS_FILE" \ + --settings "$SETTINGS_FILE" \ + $MAVEN_OPTIONS cd "${BUILD_DIR}" || exit 1 done @@ -109,10 +112,14 @@ if [ "${distribution_status}" == "not_included" ]; then git clone "https://git.opendaylight.org/gerrit/p/integration/distribution" cd distribution || exit 1 git checkout "${DISTRIBUTION_BRANCH_TO_BUILD}" + # Build project + "$MVN" clean install \ + -e -Pq \ + -Dstream="$DISTROSTREAM" \ + -DskipTests=true \ + --global-settings "$GLOBAL_SETTINGS_FILE" \ + --settings "$SETTINGS_FILE" \ + $MAVEN_OPTIONS cd "${BUILD_DIR}" || exit 1 - echo "distribution" >> ${POM_FILE} fi -# finish pom file -echo "" >> "${POM_FILE}" -echo "" >> "${POM_FILE}" diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index bc9ebb8f9..1a69611b4 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -82,7 +82,8 @@ name: PATCHES_TO_BUILD default: '' description: | - Patches to add to distro in CSV project:changeset format (genius:32/53632/9,netvirt:59/50259/47) + Ordered list of patches to build in the format of project=changeset (checkout) or + project:changeset (cherry-pick) for example: genius=32/53632/9,netvirt:59/50259/47 - parameter: name: run-test-parameter -- 2.36.6