X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Fautorelease%2Finclude-raw-autorelease-notify-build-failure.sh;h=ca31d529c5d3f0bd25894c74f52c4794d0dd29bd;hb=18fa5eebf09dea54acab41851041d64ad1551127;hp=028911076d17fe4a9b80c7a8d762fb6175992727;hpb=b134f6be261ef866380f405bd717884776842928;p=releng%2Fbuilder.git diff --git a/jjb/autorelease/include-raw-autorelease-notify-build-failure.sh b/jjb/autorelease/include-raw-autorelease-notify-build-failure.sh index 028911076..ca31d529c 100644 --- a/jjb/autorelease/include-raw-autorelease-notify-build-failure.sh +++ b/jjb/autorelease/include-raw-autorelease-notify-build-failure.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # @License EPL-1.0 ############################################################################## # Copyright (c) 2017 The Linux Foundation and others. @@ -9,59 +9,146 @@ # http://www.eclipse.org/legal/epl-v10.html ############################################################################## -#RELEASE_EMAIL="release@lists.opendaylight.org" -RELEASE_EMAIL="abelur@linuxfoundation.org, thanh.ha@linuxfoundation.org" +NEWLINE=$'\n' +RELEASE_EMAIL="release@lists.opendaylight.org" ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER" CONSOLE_LOG="/tmp/autorelease-build.log" +STREAM=${JOB_NAME#*-*e-} +ERROR_LOG="$WORKSPACE/error.log.gz" -BODY="Please refer to the logs server URL for console logs when possible -and use the Jenkins Build URL as a last resort. +# get console logs +wget --no-verbose -O "$CONSOLE_LOG" "${BUILD_URL}consoleText" -Console Logs URL: -https://logs.opendaylight.org/$SILO/$ARCHIVES_DIR +# TODO: This section is still required since some of the projects use +# description. Remove this section when the reactor info is more consistant. +# extract failing project from reactor information +REACTOR_INFO=$(awk '/Reactor Summary:/ { flag=1 } + flag { + if ( sub(/^\[(INFO)\]/,"") && sub(/FAILURE \[.*/,"") ) { + gsub(/[[:space:]]*::[[:space:]]*/,"::") + gsub(/^[[:space:]]+|[[:space:]]+$|[.]/,"") + print + } + } + /Final Memory:/ { flag=0 }' $CONSOLE_LOG) -Jenkins Build URL: -$BUILD_URL" +# check for project format +if [[ ${REACTOR_INFO} =~ .*::*.*::*. ]]; then + # extract project and artifactId from full format + ODL=$(echo "${REACTOR_INFO}" | awk -F'::' '{ gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 }') + PROJECT_=$(echo "${REACTOR_INFO}" | awk -F'::' '{ gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2 }') + NAME=$(echo "${REACTOR_INFO}" | awk -F'::' '{ gsub(/^[ \t]+|[ \t]+$/, "", $3); print $3 }') +else + # set project from partial format + ODL="" + PROJECT_="" + NAME=$(echo "${REACTOR_INFO}" | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }') +fi -# get console logs -wget -O $CONSOLE_LOG ${BUILD_URL}consoleText +# determine ARTIFACT_ID for project mailing list +ARTIFACT_ID=$(awk -F: '/\[ERROR\].*mvn -rf :/ { print $2}' $CONSOLE_LOG) -# get the failed project or artifactid -TEMP=`awk '/Reactor Summary:/{flag=1;next} \ - /Final Memory:/{flag=0}flag' $CONSOLE_LOGS \ - | grep '. FAILURE \[' | awk -F'[].]' '{gsub(/ /, "", $2); print $2 }'` +# determine project mailing list using xpaths +# if project.groupId: +# project.groupId is set and is not inherited +# else if project.parent.groupId: +# project.groupId is not set but IS inherited from project.parent.groupId +# else +# exclude project mailing list +if [ ! -z "$ARTIFACT_ID" ]; then + grouplist=() + while IFS="" read -r p; do + GROUP=$(xmlstarlet sel\ + -N "x=http://maven.apache.org/POM/4.0.0"\ + -t -m "/x:project[x:artifactId='$ARTIFACT_ID']"\ + --if "/x:project/x:groupId"\ + -v "/x:project/x:groupId"\ + --elif "/x:project/x:parent/x:groupId"\ + -v "/x:project/x:parent/x:groupId"\ + --else -o ""\ + "$p" 2>/dev/null) + if [ ! -z "${GROUP}" ]; then + grouplist+=($(echo "${GROUP}" | awk -F'.' '{ print $3 }')) + fi + done < <(find . -name "pom.xml") -# check for project format -if [[ ${TEMP} =~ .*::*.*::*. ]]; then - # extract project and artifactid from full format - PROJECT=`echo ${TEMP} | awk -F'::' '{ print $2 }'` - ARTIFACTID=`echo ${TEMP} |awk -F'::' '{ print $3 }'` + if [ "${#grouplist[@]}" -eq 1 ]; then + PROJECT="${grouplist[0]}" + elif [ "${#grouplist[@]}" -gt 1 ]; then + GROUPLIST="NOTE: The artifactId: $ARTIFACT_ID matches multiple groups: ${grouplist[*]}" + else + echo "Failed to determine project.groupId using xpaths" + fi else - # set ARTIFACTID to partial format - ARTIFACTID=${TEMP} + echo "Failed to determine ARTIFACT_ID" + exit 1 fi +# Construct email subject & body +PROJECT_STRING=${PROJECT:+" from $PROJECT"} +SUBJECT="[release] Autorelease $STREAM failed to build $ARTIFACT_ID$PROJECT_STRING" +# shellcheck disable=SC2034 +ATTACHMENT_INCLUDE="Attached is a snippet of the error message related to the +failure that we were able to automatically parse as well as console logs." +# shellcheck disable=SC2034 +ATTACHMENT_EXCLUDE="Unable to attach error message snippet related to the failure +since this exceeds the mail server attachment size limit. Please +refer ${ERROR_LOG##*/} in archives directory." +ATTACHMENT=ATTACHMENT_INCLUDE # default behaviour +BODY="Attention ${PROJECT:-"OpenDaylight"}-devs, + +Autorelease $STREAM failed to build $ARTIFACT_ID$PROJECT_STRING in build +$BUILD_NUMBER. \${!ATTACHMENT} ${PROJECT:+${NEWLINE}${GROUPLIST}} + +Console Logs: +https://logs.opendaylight.org/$SILO/$ARCHIVES_DIR + +Jenkins Build: +$BUILD_URL + +Please review and provide an ETA on when a fix will be available. + +Thanks, +ODL releng/autorelease team +" + # check if remote staging is complete successfully -BUILD_STATUS=`awk '/\[INFO\] Remote staging finished/{flag=1;next} \ - /Total time:/{flag=0}flag' $CONSOLE_LOG \ - | grep '\] BUILD' | awk '{print $3}'` +BUILD_STATUS=$(awk '/\[INFO\] Remote staging finished/{flag=1;next}/Total time:/{flag=0}flag' $CONSOLE_LOG \ + | grep '\] BUILD' | awk '{print $3}') -if [ ! -z "${ARTIFACTID}" ] && [[ "${BUILD_STATUS}" != "SUCCESS" ]]; then +if ([ ! -z "${NAME}" ] || [ ! -z "${ARTIFACT_ID}" ]) && [[ "${BUILD_STATUS}" != "SUCCESS" ]]; then # project search pattern should handle both scenarios - # 1. Full format: ODL :: $PROJECT :: $ARTIFACTID - # 2. Partial format: Building $ARTIFACTID - awk "/\[INFO\] Building ${ARTIFACTID} / || /ODL :: ${PROJECT} :: ${ARTIFACTID} /{flag=1;next} \ - /Reactor Summary:/{flag=0}flag" $CONSOLE_LOG > /tmp/error_msg - - if [ -z "${PROJECT}" ]; then - PROJECT=${ARTIFACTID} - # TODO: unset the below line when ready to deploy to real lists - # RELEASE_EMAIL = "${RELEASE_EMAIL}, ${PROJECT}-dev@opendaylight.org" + # 1. Full format: ODL :: $PROJECT :: $ARTIFACT_ID + # 2. Partial format: Building $ARTIFACT_ID + sed -e "/\[INFO\] Building \(${NAME} \|${ARTIFACT_ID} \|${ODL} :: ${PROJECT_} :: ${NAME} \)/,/Reactor Summary:/!d;//d" \ + $CONSOLE_LOG | gzip > "$ERROR_LOG" + + if [ -n "${PROJECT}" ]; then + RELEASE_EMAIL="${RELEASE_EMAIL}, ${PROJECT}-dev@lists.opendaylight.org" fi - SUBJECT="[release] Autorelease build failure: ${PROJECT}" + file_size=$(du -k "$ERROR_LOG" | cut -f1) - echo "${BODY}" | mail -A /tmp/error_msg -s "${SUBJECT}" "${RELEASE_EMAIL}" + # Only send emails in production (releng), not testing (sandbox) + if [ "${SILO}" == "releng" ]; then + if [[ "$file_size" -gt 100 ]]; then + # shellcheck disable=SC2034 + ATTACHMENT=ATTACHMENT_EXCLUDE + eval echo \""${BODY}"\" | mail \ + -r "Jenkins " \ + -s "${SUBJECT}" "${RELEASE_EMAIL}" + else + eval echo \""${BODY}"\" | mail -a "$ERROR_LOG" \ + -r "Jenkins " \ + -s "${SUBJECT}" "${RELEASE_EMAIL}" + fi + elif [ "${SILO}" == "sandbox" ]; then + echo "Running in sandbox, not actually sending notification emails" + echo "Subject: ${SUBJECT}" + eval echo \""Body: ${BODY}"\" + else + echo "Not sure how to notify in \"${SILO}\" Jenkins silo" + fi fi rm $CONSOLE_LOG