From 349cdbdfce4522d3949d635ac075908e7296198f Mon Sep 17 00:00:00 2001 From: Anil Belur Date: Thu, 1 Feb 2024 10:03:02 +1000 Subject: [PATCH] Feat!: Update JJB version to 6x Jenkins Job Builder 6.0.0 is released and breaks backward compatibility with 5.1.0. The breaking changes with previous 5.1.0 release: - Jobs are now expanded the same way as job templates. - Macros without parameters are now expanded the same way as macros with parameters. - Tags !include-raw: and !include-raw-escape: should now be used the same way in jobs and macros without parameters as they are used in job templates and macros with parameters. - Fix shellcheck warnings Ref: https://storyboard.openstack.org/#!/story/2010963 https://storyboard.openstack.org/#!/story/2010588 Issue: LF-JIRA RELENG-5119 Signed-off-by: Anil Belur Change-Id: I9c16d62bf40c1396a630b479dca12a62a6e48c56 Signed-off-by: Anil Belur --- global-jjb | 2 +- .../autorelease-checkout-gerrit-patch.sh | 18 +++++ jjb/autorelease/autorelease-macros.yaml | 32 ++------- jjb/docs/docs-macros.yaml | 2 +- .../distribution-check-verify-groupid.sh | 28 ++++++++ .../distribution/distribution-check-wipe.sh | 39 +++++++++++ .../distribution/distribution-macros.yaml | 46 +------------ .../distribution/distribution-templates.yaml | 8 ++- .../integration-csit-archive-artifacts.sh | 21 ++++++ .../integration-csit-run-postscript.sh | 13 ++++ jjb/integration/integration-macros.yaml | 68 ++++++------------- jjb/releng-macros.yaml | 2 +- jjb/releng-templates-java.yaml | 8 ++- 13 files changed, 164 insertions(+), 123 deletions(-) create mode 100644 jjb/autorelease/autorelease-checkout-gerrit-patch.sh create mode 100644 jjb/integration/distribution/distribution-check-verify-groupid.sh create mode 100644 jjb/integration/distribution/distribution-check-wipe.sh create mode 100644 jjb/integration/integration-csit-archive-artifacts.sh create mode 100644 jjb/integration/integration-csit-run-postscript.sh diff --git a/global-jjb b/global-jjb index 766dbef8a..5d336574e 160000 --- a/global-jjb +++ b/global-jjb @@ -1 +1 @@ -Subproject commit 766dbef8a63d824d6e551d4f179806521be95d13 +Subproject commit 5d336574e77fb349ace15356ef8ffb816d463544 diff --git a/jjb/autorelease/autorelease-checkout-gerrit-patch.sh b/jjb/autorelease/autorelease-checkout-gerrit-patch.sh new file mode 100644 index 000000000..9e83fb09e --- /dev/null +++ b/jjb/autorelease/autorelease-checkout-gerrit-patch.sh @@ -0,0 +1,18 @@ +#!/bin/bash -l +set -eu -o pipefail + +# If the project is autorelease then we do not need to cd +if [ "$GERRIT_PROJECT" != "releng/autorelease" ]; then + cd "$WORKSPACE/$GERRIT_PROJECT" +fi + +echo "Checking out ${GERRIT_PROJECT} patch ${GERRIT_REFSPEC}..." +git fetch origin "${GERRIT_REFSPEC}" && git checkout FETCH_HEAD + +# If the project is autorelease then we need to init and update submodules +if [ "$GERRIT_PROJECT" == "releng/autorelease" ]; then + git submodule update --init + # The previous checkout might have failed to remove directory of a submodule being removed. + # See https://stackoverflow.com/a/10761699 + git clean -dff +fi diff --git a/jjb/autorelease/autorelease-macros.yaml b/jjb/autorelease/autorelease-macros.yaml index 2cc67e01d..81990930e 100644 --- a/jjb/autorelease/autorelease-macros.yaml +++ b/jjb/autorelease/autorelease-macros.yaml @@ -19,25 +19,7 @@ - builder: name: autorelease-checkout-gerrit-patch builders: - - shell: | - #!/bin/bash -l - set -eu -o pipefail - - # If the project is autorelease then we do not need to cd - if [ "$GERRIT_PROJECT" != "releng/autorelease" ]; then - cd "$WORKSPACE/$GERRIT_PROJECT" - fi - - echo "Checking out ${GERRIT_PROJECT} patch ${GERRIT_REFSPEC}..." - git fetch origin ${GERRIT_REFSPEC} && git checkout FETCH_HEAD - - # If the project is autorelease then we need to init and update submodules - if [ "$GERRIT_PROJECT" == "releng/autorelease" ]; then - git submodule update --init - # The previous checkout might have failed to remove directory of a submodule being removed. - # See https://stackoverflow.com/a/10761699 - git clean -dff - fi + - shell: !include-raw-escape: autorelease-checkout-gerrit-patch.sh - builder: name: autorelease-determine-merge-order @@ -103,14 +85,14 @@ - builder: name: autorelease-generate-release-patches builders: - - shell: !include-raw: + - shell: !include-raw-escape: - prepare-release.sh - builder: name: autorelease-get-integration-test-variables builders: - integration-set-variables - - shell: !include-raw: autorelease-get-integration-test-variables.sh + - shell: !include-raw-escape: autorelease-get-integration-test-variables.sh - inject: properties-file: variables.prop @@ -123,18 +105,18 @@ - builder: name: autorelease-generate-release-notes builders: - - shell: !include-raw: + - shell: !include-raw-escape: - generate-release-notes.sh - builder: name: autorelease-update-project-views builders: - - shell: !include-raw: update-autorelease-projects-views.sh + - shell: !include-raw-escape: update-autorelease-projects-views.sh - builder: name: autorelease-update-validate-projects builders: - - shell: !include-raw: update-validate-autorelease-projects.sh + - shell: !include-raw-escape: update-validate-autorelease-projects.sh - builder: name: autorelease-compare-projects @@ -155,5 +137,5 @@ - SUCCESS - UNSTABLE build-steps: - - shell: !include-raw: notify-build-failure.sh + - shell: !include-raw-escape: notify-build-failure.sh mark-unstable-if-failed: true diff --git a/jjb/docs/docs-macros.yaml b/jjb/docs/docs-macros.yaml index 0dd07a6e3..59e7c392e 100644 --- a/jjb/docs/docs-macros.yaml +++ b/jjb/docs/docs-macros.yaml @@ -3,6 +3,6 @@ - builder: name: docs-update builders: - - shell: !include-raw: + - shell: !include-raw-escape: - docs_version_bump.sh - docs_version_bump_projects.sh diff --git a/jjb/integration/distribution/distribution-check-verify-groupid.sh b/jjb/integration/distribution/distribution-check-verify-groupid.sh new file mode 100644 index 000000000..9761dd143 --- /dev/null +++ b/jjb/integration/distribution/distribution-check-verify-groupid.sh @@ -0,0 +1,28 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2024 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script performs version bumping activities for an ODL release. +echo "---> distribution-check-verify-groupid.sh" + +echo "These are allowed projects: $ALLOW_PROJECTS" +echo "These are distribution pulled projects:" +EXIT_CODE="0" +for PROJECT in /tmp/r/org/opendaylight/*; do + if [[ ! -d "$PROJECT" ]]; then + continue + fi + echo "Checking $PROJECT" + if [[ "$ALLOW_PROJECTS" != *"$PROJECT"* ]]; then + echo "ERROR: Not allowed project $PROJECT pulled" + EXIT_CODE="1" + fi +done +exit $EXIT_CODE diff --git a/jjb/integration/distribution/distribution-check-wipe.sh b/jjb/integration/distribution/distribution-check-wipe.sh new file mode 100644 index 000000000..c7f552d05 --- /dev/null +++ b/jjb/integration/distribution/distribution-check-wipe.sh @@ -0,0 +1,39 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2024 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script performs version bumping activities for an ODL release. +echo "---> distribution-check-wipe.sh" + +echo "wipe r: the local Maven repository" +rm -rfv /tmp/r +echo "wipe n: the fake remote (Nexus) repository" +rm -rfv /tmp/n +echo "wipe t: the transient repository used in some manipulations" +rm -rfv /tmp/t +echo "create n: multithreaded execution might fail at creating it." +mkdir /tmp/n +echo "detecting distribution allowed projects" +# Some allowed projects cannot be detected in distribution because they do not produce features. +ALLOW_PROJECTS=(yangtools mdsal openflowjava) +if [[ "$KARAF_VERSION" == "odl" ]]; then +# shellcheck disable=SC2207 +ALLOW_PROJECTS+=($(grep 'org.opendaylight.' -Rh distribution \ +| sed -e 's%^[ \t]*org.opendaylight.%%' \ +| sed -e 's%%%' | sort -u)) +else +# For Managed distro we only look at the features folder +# shellcheck disable=SC2207 +ALLOW_PROJECTS+=($(grep 'org.opendaylight.' -Rh distribution/features \ +| sed -e 's%^[ \t]*org.opendaylight.%%' \ +| sed -e 's%%%' | sort -u)) +fi +echo "Allowed projects are " "${ALLOW_PROJECTS[@]}" +echo "ALLOW_PROJECTS= " "${ALLOW_PROJECTS[@]}" > allowed_projects.txt diff --git a/jjb/integration/distribution/distribution-macros.yaml b/jjb/integration/distribution/distribution-macros.yaml index 974382e2f..3b0091d60 100644 --- a/jjb/integration/distribution/distribution-macros.yaml +++ b/jjb/integration/distribution/distribution-macros.yaml @@ -4,30 +4,7 @@ name: distribution-check-wipe # Step zero: Wipe file repositories up front. builders: - - shell: | - echo "wipe r: the local Maven repository" - rm -rfv /tmp/r - echo "wipe n: the fake remote (Nexus) repository" - rm -rfv /tmp/n - echo "wipe t: the transient repository used in some manipulations" - rm -rfv /tmp/t - echo "create n: multithreaded execution might fail at creating it." - mkdir /tmp/n - echo "detecting distribution allowed projects" - # Some allowed projects cannot be detected in distribution because they do not produce features. - ALLOW_PROJECTS=(yangtools mdsal openflowjava) - if [[ "$KARAF_VERSION" == "odl" ]]; then - ALLOW_PROJECTS+=(`grep 'org.opendaylight.' -Rh distribution \ - | sed -e 's%^[ \t]*org.opendaylight.%%' \ - | sed -e 's%%%' | sort -u`) - else - # For Managed distro we only look at the features folder - ALLOW_PROJECTS+=(`grep 'org.opendaylight.' -Rh distribution/features \ - | sed -e 's%^[ \t]*org.opendaylight.%%' \ - | sed -e 's%%%' | sort -u`) - fi - echo "Allowed projects are ${ALLOW_PROJECTS[@]}" - echo "ALLOW_PROJECTS=${ALLOW_PROJECTS[@]}" > allowed_projects.txt + - shell: !include-raw-escape: distribution-check-wipe.sh - builder: name: distribution-check-build-project @@ -61,24 +38,7 @@ # Arguments: # gerrit-project: Project name as nexus URI part. Typically '$GERRIT_PROJECT'. builders: - - shell: | - echo "These are allowed projects: $ALLOW_PROJECTS" - echo "These are distribution pulled projects:" - EXIT_CODE="0" - for PROJECT in `ls /tmp/r/org/opendaylight`; do - echo "checking $PROJECT" - if [[ "$ALLOW_PROJECTS" != *"$PROJECT"* ]]; then - echo "ERROR: Not allowed project $PROJECT pulled" - EXIT_CODE="1" - fi - done - exit $EXIT_CODE - echo "verify project groupId" - mkdir -p /tmp/t/org/opendaylight/{gerrit-project} - mv /tmp/n/org/opendaylight/{gerrit-project}/* /tmp/t/org/opendaylight/{gerrit-project}/ - test -z "`find /tmp/n/ -type f`" || ( echo "ERROR: Mismatched groupId detected (see above)." && false ) - rm -rf /tmp/n - mv /tmp/t /tmp/n + - shell: !include-raw-escape: distribution-check-verify-groupid.sh - builder: name: distribution-check-download-deps @@ -277,4 +237,4 @@ # KARAF_VERSION: 'karaf4' or 'karaf3'. # Use distribution-karaf-version macro to customize. builders: - - shell: !include-raw: distribution-check-bootup.sh + - shell: !include-raw-escape: distribution-check-bootup.sh diff --git a/jjb/integration/distribution/distribution-templates.yaml b/jjb/integration/distribution/distribution-templates.yaml index 507752b48..6baad6847 100644 --- a/jjb/integration/distribution/distribution-templates.yaml +++ b/jjb/integration/distribution/distribution-templates.yaml @@ -132,6 +132,7 @@ name: "distribution-verify-{type}-{stream}" disable-job: "false" + jacoco-exclude-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**" project-type: freestyle node: centos8-builder-8c-8g @@ -193,7 +194,8 @@ publishers: - findbugs - - lf-jacoco-report + - lf-jacoco-report: + exclude-pattern: { jacoco-exclude-pattern } - lf-infra-publish - email-notification: email-recipients: "{email-recipients}" @@ -208,6 +210,7 @@ # https://issues.apache.org/jira/browse/MNG-5666 disable-job: false + jacoco-exclude-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**" project-type: maven node: "centos8-builder-8c-8g" @@ -287,7 +290,8 @@ id: "" unique-version: true deploy-unstable: false - - lf-jacoco-report + - lf-jacoco-report: + exclude-pattern: "{jacoco-exclude-pattern}" - lf-infra-publish - email-notification: email-recipients: "{email-recipients}" diff --git a/jjb/integration/integration-csit-archive-artifacts.sh b/jjb/integration/integration-csit-archive-artifacts.sh new file mode 100644 index 000000000..95f4fd081 --- /dev/null +++ b/jjb/integration/integration-csit-archive-artifacts.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +echo "Archiving csit artifacts" +cd "$WORKSPACE" || exit +mkdir -p ./archives +for i in $(seq 1 "${NUM_ODL_SYSTEM}"); do + NODE_FOLDER="./archives/odl_${i}" + mkdir -p "${NODE_FOLDER}" + mv odl"${i}"_* "${NODE_FOLDER}" || true + mv karaf_"${i}"_*_threads* "${NODE_FOLDER}" || true + mv -- *_"${i}".png "${NODE_FOLDER}" || true + mv /tmp/odl"${i}"_* "${NODE_FOLDER}" || true + mv gclogs-"${i}" "${NODE_FOLDER}" || true +done +curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip" +unzip -d ./archives robot-plugin.zip +mv -- *.log *.log.gz *.csv *.png ./archives || true # Don't fail if file missing +# TODO: Can the following line ever fail? +find . -type f -name '*.hprof' -print0 \ + | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz +# TODO: Tweak the compression level if better ratio (or speed) is needed. diff --git a/jjb/integration/integration-csit-run-postscript.sh b/jjb/integration/integration-csit-run-postscript.sh new file mode 100644 index 000000000..38fcbf65f --- /dev/null +++ b/jjb/integration/integration-csit-run-postscript.sh @@ -0,0 +1,13 @@ +#!/bin/bash +#The following script runs any configurable postplan stored in test/csit/postplans. +if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then + echo "postplan exists!!!" + echo "Changing the postplan path..." + script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN} + sed "s:integration:${WORKSPACE}:" "${script_name}" > postplan.txt + cat postplan.txt + grep -Ev '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do + echo "Executing ${line}..." + ${line} + done +fi diff --git a/jjb/integration/integration-macros.yaml b/jjb/integration/integration-macros.yaml index 3043e4b59..f874e911e 100644 --- a/jjb/integration/integration-macros.yaml +++ b/jjb/integration/integration-macros.yaml @@ -293,7 +293,7 @@ - builder: name: integration-get-slave-addresses builders: - - shell: !include-raw: integration-get-slave-addresses.sh + - shell: !include-raw-escape: integration-get-slave-addresses.sh - builder: name: integration-set-variables @@ -301,7 +301,7 @@ # Required bash variables: # KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set. builders: - - shell: !include-raw: integration-set-variables.sh + - shell: !include-raw-escape: integration-set-variables.sh - inject: properties-file: set_variables.env @@ -313,7 +313,7 @@ # KARAF_VERSION: 'karaf4' or 'karaf3'. Use distribution-karaf-version macro to set. builders: - integration-set-variables - - shell: !include-raw: integration-detect-variables.sh + - shell: !include-raw-escape: integration-detect-variables.sh - inject: properties-file: detect_variables.env @@ -324,7 +324,7 @@ - builder: name: integration-deploy-controller-run-test builders: - - shell: !include-raw: integration-deploy-controller-run-test.sh + - shell: !include-raw-escape: integration-deploy-controller-run-test.sh # Macro: integration-deply-non-karaf-controller-run-test # Operation: this macro deploys non-karaf controller application and run test @@ -333,7 +333,7 @@ - builder: name: integration-run-test builders: - - shell: !include-raw: integration-run-test.sh + - shell: !include-raw-escape: integration-run-test.sh # Macro: integration-configure-clustering # Operation: this macro configures the clustering @@ -342,7 +342,7 @@ - builder: name: integration-configure-clustering builders: - - shell: !include-raw: integration-configure-clustering.sh + - shell: !include-raw-escape: integration-configure-clustering.sh # Macro: integration-start-cluster-run-test # Operation: this macro starts the 3-node cluster and runs test @@ -351,7 +351,7 @@ - builder: name: integration-start-cluster-run-test builders: - - shell: !include-raw: integration-start-cluster-run-test.sh + - shell: !include-raw-escape: integration-start-cluster-run-test.sh # Macro: integration-install-robotframework # Operation: Installs robotframework using pip to a virtualenv @@ -360,7 +360,7 @@ - builder: name: integration-install-robotframework builders: - - shell: !include-raw: integration-install-robotframework.sh + - shell: !include-raw-escape: integration-install-robotframework.sh # Macro: integration-cleanup-tmp # Operation: Cleans up temporary files created by build @@ -369,7 +369,7 @@ - builder: name: integration-cleanup-tmp builders: - - shell: !include-raw: integration-cleanup-tmp.sh + - shell: !include-raw-escape: integration-cleanup-tmp.sh # Macro: integration-multipatch-builder # Operation: checks out multiple patches and builds custom distribution @@ -378,7 +378,7 @@ - builder: name: integration-multipatch-builder builders: - - shell: !include-raw: multipatch-distribution.sh + - shell: !include-raw-escape: multipatch-distribution.sh # Macro: integration-rebase-patch # Operation: For cloned project, rebase checked-out Gerrit patch onto $DISTROBRANCH. @@ -387,7 +387,7 @@ - builder: name: integration-rebase-gerrit-patch builders: - - shell: !include-raw: integration-rebase-gerrit-patch.sh + - shell: !include-raw-escape: integration-rebase-gerrit-patch.sh # Macro: integration-install-common-functions # Operation: Copy the common-functions.sh script to csit nodes @@ -395,7 +395,7 @@ - builder: name: integration-install-common-functions builders: - - shell: !include-raw: + - shell: !include-raw-escape: - copy-common-functions.sh - common-functions.sh @@ -408,7 +408,12 @@ # Required bash variables: # KARAF_ARTIFACT: 'karaf' or 'distribution-karaf'. Use integration-set-variables. builders: - - shell: !include-raw: integration-compare-distributions.sh + - shell: !include-raw-escape: integration-compare-distributions.sh + +- builder: + name: integration-csit-archive-artifacts + builders: + - shell: !include-raw-escape: integration-csit-archive-artifacts.sh - builder: name: integration-upload-distribution @@ -461,27 +466,7 @@ - SUCCESS - UNSTABLE build-steps: - - shell: | - #!/bin/bash - echo "Archiving csit artifacts" - cd $WORKSPACE - mkdir -p ./archives - for i in `seq 1 ${NUM_ODL_SYSTEM}`; do - NODE_FOLDER="./archives/odl_${i}" - mkdir -p ${NODE_FOLDER} - mv odl${i}_* ${NODE_FOLDER} || true - mv karaf_${i}_*_threads* ${NODE_FOLDER} || true - mv *_${i}.png ${NODE_FOLDER} || true - mv /tmp/odl${i}_* ${NODE_FOLDER} || true - mv gclogs-${i} ${NODE_FOLDER} || true - done - curl --output robot-plugin.zip "$BUILD_URL/robot/report/*zip*/robot-plugin.zip" - unzip -d ./archives robot-plugin.zip - mv *.log *.log.gz *.csv *.png ./archives || true # Don't fail if file missing - # TODO: Can the following line ever fail? - find . -type f -name *.hprof -print0 \ - | tar -cvf - --null -T - | xz --threads=0 > ./archives/hprof.tar.xz - # TODO: Tweak the compression level if better ratio (or speed) is needed. + - shell: !include-raw-escape: integration-csit-archive-artifacts.sh mark-unstable-if-failed: true - publisher: @@ -497,18 +482,5 @@ - SUCCESS - UNSTABLE build-steps: - - shell: | - #!/bin/bash - #The following script runs any configurable postplan stored in test/csit/postplans. - if [ -f "${WORKSPACE}/test/csit/postplans/${TESTPLAN}" ]; then - echo "postplan exists!!!" - echo "Changing the postplan path..." - script_name=${WORKSPACE}/test/csit/postplans/${TESTPLAN} - cat ${script_name} | sed "s:integration:${WORKSPACE}:" > postplan.txt - cat postplan.txt - egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' postplan.txt | while read -r line ; do - echo "Executing ${line}..." - ${line} - done - fi + - shell: !include-raw-escape: integration-csit-run-postscript.sh mark-unstable-if-failed: true diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index 48355d67b..cc4bfe4d7 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -284,5 +284,5 @@ - SUCCESS - UNSTABLE build-steps: - - shell: !include-raw: generate-csit-status-report.sh + - shell: !include-raw-escape: generate-csit-status-report.sh mark-unstable-if-failed: true diff --git a/jjb/releng-templates-java.yaml b/jjb/releng-templates-java.yaml index 3d5074e70..d804eac11 100644 --- a/jjb/releng-templates-java.yaml +++ b/jjb/releng-templates-java.yaml @@ -98,6 +98,7 @@ **/target/surefire-reports/*-output.txt mvn-goals: clean install disable-job: false + jacoco-exclude-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**" ##################### # Job Configuration # @@ -156,7 +157,8 @@ email-recipients: "{email-recipients}" email-prefix: "{email-upstream}" - findbugs - - lf-jacoco-report + - lf-jacoco-report: + exclude-pattern: "{jacoco-exclude-pattern}" - lf-infra-publish - job-template: @@ -166,6 +168,7 @@ # Default parameters # ###################### disable-job: "false" + jacoco-exclude-pattern: "**/gen/**,**/generated-sources/**,**/yang-gen**,**/pax/**" archive-artifacts: > **/*.log @@ -258,5 +261,6 @@ id: "" unique-version: true deploy-unstable: false - - lf-jacoco-report + - lf-jacoco-report: + exclude-pattern: "{jacoco-exclude-pattern}" - lf-infra-publish -- 2.36.6