jjb/autorelease/notify-build-failure.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: EPL-1.0
   3 ##############################################################################
   4 # Copyright (c) 2017 The Linux Foundation and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Eclipse Public License v1.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.eclipse.org/legal/epl-v10.html
  10 ##############################################################################
  11 # This script parses the console log to discover which project has caused a
  12 # build failure and notify the project as appropriate.
  13
  14 echo "---> notify-build-failure.sh"
  15
  16 NEWLINE=$'\n'
  17 RELEASE_EMAIL="release@lists.opendaylight.org"
  18 ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER"
  19 CONSOLE_LOG="/tmp/autorelease-build.log"
  20 STREAM=${JOB_NAME#*-*e-}
  21 ERROR_LOG="$WORKSPACE/archives/error.log.gz"
  22
  23 wget -nv -O "$CONSOLE_LOG" "${BUILD_URL}consoleText"
  24
  25 # TODO: This section is still required since some of the projects use
  26 # description. Remove this section when the reactor info is more consistant.
  27 # extract failing project from reactor information
  28 REACTOR_INFO=$(awk '/Reactor Summary:/ { flag=1 }
  29           flag {
  30              if ( sub(/^\[(INFO)\]/,"") && sub(/FAILURE \[.*/,"") ) {
  31                  gsub(/[[:space:]]*::[[:space:]]*/,"::")
  32                  gsub(/^[[:space:]]+|[[:space:]]+$|[.]/,"")
  33                  print
  34              }
  35           }
  36           /Final Memory:/ { flag=0 }' $CONSOLE_LOG)
  37
  38 # check for project format
  39 if [[ ${REACTOR_INFO} =~ .*::*.*::*. ]]; then
  40     # extract project and artifactId from full format
  41     ODL=$(echo "${REACTOR_INFO}" | awk -F'::' '{ gsub(/^[ \t]+|[ \t]+$/, "", $1); print $1 }')
  42     PROJECT_=$(echo "${REACTOR_INFO}" | awk -F'::' '{ gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2 }')
  43     NAME=$(echo "${REACTOR_INFO}" | awk -F'::' '{ gsub(/^[ \t]+|[ \t]+$/, "", $3); print $3 }')
  44 else
  45     # set project from partial format
  46     ODL=""
  47     PROJECT_=""
  48     NAME=$(echo "${REACTOR_INFO}" | awk '{ gsub(/^[ \t]+|[ \t]+$/, ""); print }')
  49 fi
  50
  51 # determine ARTIFACT_ID for project mailing list
  52 ARTIFACT_ID=$(awk -F: '/\[ERROR\].*mvn <goals> -rf :/ { print $2}' $CONSOLE_LOG)
  53
  54 # determine project mailing list using xpaths
  55 # if project.groupId:
  56 #     project.groupId is set and is not inherited
  57 # else if project.parent.groupId:
  58 #     project.groupId is not set but IS inherited from project.parent.groupId
  59 # else
  60 #     exclude project mailing list
  61 if [ ! -z "$ARTIFACT_ID" ]; then
  62     grouplist=()
  63     while IFS="" read -r p; do
  64         GROUP=$(xmlstarlet sel\
  65               -N "x=http://maven.apache.org/POM/4.0.0"\
  66               -t -m "/x:project[x:artifactId='$ARTIFACT_ID']"\
  67               --if "/x:project/x:groupId"\
  68               -v "/x:project/x:groupId"\
  69               --elif "/x:project/x:parent/x:groupId"\
  70               -v "/x:project/x:parent/x:groupId"\
  71               --else -o ""\
  72               "$p" 2>/dev/null)
  73         if [ ! -z "${GROUP}" ]; then
  74             # shellcheck disable=SC2207
  75             grouplist+=($(echo "${GROUP}" | awk -F'.' '{ print $3 }'))
  76         fi
  77     done < <(find . -name "pom.xml")
  78
  79     if [ "${#grouplist[@]}" -eq 1 ]; then
  80         PROJECT="${grouplist[0]}"
  81     elif [ "${#grouplist[@]}" -gt 1 ]; then
  82         GROUPLIST="NOTE: The artifactId: $ARTIFACT_ID matches multiple groups: ${grouplist[*]}"
  83     else
  84         echo "Failed to determine project.groupId using xpaths"
  85     fi
  86 else
  87     echo "Failed to determine ARTIFACT_ID"
  88     exit 1
  89 fi
  90
  91 # Construct email subject & body
  92 PROJECT_STRING=${PROJECT:+" from $PROJECT"}
  93 SUBJECT="[release] Autorelease $STREAM failed to build $ARTIFACT_ID$PROJECT_STRING"
  94 # shellcheck disable=SC2034
  95 ATTACHMENT_INCLUDE="Attached is a snippet of the error message related to the
  96 failure that we were able to automatically parse as well as console logs."
  97 # shellcheck disable=SC2034
  98 ATTACHMENT_EXCLUDE="Unable to attach error message snippet related to the failure
  99 since this exceeds the mail server attachment size limit. Please
 100 refer ${ERROR_LOG##*/} in archives directory."
 101 ATTACHMENT=ATTACHMENT_INCLUDE  # default behaviour
 102 BODY="Attention ${PROJECT:-"OpenDaylight"}-devs,
 103
 104 Autorelease $STREAM failed to build $ARTIFACT_ID$PROJECT_STRING in build
 105 $BUILD_NUMBER. \${!ATTACHMENT} ${PROJECT:+${NEWLINE}${GROUPLIST}}
 106
 107 Console Logs:
 108 https://logs.opendaylight.org/$SILO/$ARCHIVES_DIR
 109
 110 Jenkins Build:
 111 $BUILD_URL
 112
 113 Please review and provide an ETA on when a fix will be available.
 114
 115 Thanks,
 116 ODL releng/autorelease team
 117 "
 118
 119 # check if remote staging is complete successfully
 120 BUILD_STATUS=$(awk '/\[INFO\] Remote staging finished/{flag=1;next}/Total time:/{flag=0}flag' $CONSOLE_LOG \
 121                    | grep '\] BUILD' | awk '{print $3}')
 122
 123 if ([ ! -z "${NAME}" ] || [ ! -z "${ARTIFACT_ID}" ]) && [[ "${BUILD_STATUS}" != "SUCCESS" ]]; then
 124     # project search pattern should handle both scenarios
 125     # 1. Full format:    ODL :: $PROJECT :: $ARTIFACT_ID
 126     # 2. Partial format: Building $ARTIFACT_ID
 127     sed -e "/\[INFO\] Building \(${NAME} \|${ARTIFACT_ID} \|${ODL} :: ${PROJECT_} :: ${NAME} \)/,/Reactor Summary:/!d;//d" \
 128           $CONSOLE_LOG | gzip > "$ERROR_LOG"
 129
 130     if [ -n "${PROJECT}" ]; then
 131         RELEASE_EMAIL="${RELEASE_EMAIL}, ${PROJECT}-dev@lists.opendaylight.org"
 132     fi
 133
 134     file_size=$(du -k "$ERROR_LOG" | cut -f1)
 135
 136     # Only send emails in production (releng), not testing (sandbox)
 137     if [ "${SILO}" == "releng" ]; then
 138         if [[ "$file_size" -gt 100 ]]; then
 139             # shellcheck disable=SC2034
 140             ATTACHMENT=ATTACHMENT_EXCLUDE
 141             eval echo \""${BODY}"\" | mail \
 142                 -r "Jenkins <jenkins-dontreply@opendaylight.org>" \
 143                 -s "${SUBJECT}" "${RELEASE_EMAIL}"
 144         else
 145             eval echo \""${BODY}"\" | mail -a "$ERROR_LOG" \
 146                 -r "Jenkins <jenkins-dontreply@opendaylight.org>" \
 147                 -s "${SUBJECT}" "${RELEASE_EMAIL}"
 148         fi
 149     elif [ "${SILO}" == "sandbox" ]; then
 150         echo "Running in sandbox, not actually sending notification emails"
 151         echo "Subject: ${SUBJECT}"
 152         eval echo \""Body: ${BODY}"\"
 153     else
 154         echo "Not sure how to notify in \"${SILO}\" Jenkins silo"
 155     fi
 156 fi
 157
 158 rm $CONSOLE_LOG