jjb/integration/distribution/distribution-check-bootup.sh

   1 #!/bin/bash
   2 if [[ -n "${CONTROLLERFEATURES}" ]]; then
   3     ACTUALFEATURES="odl-integration-all,${CONTROLLERFEATURES}"
   4 else
   5     ACTUALFEATURES="odl-integration-all"
   6 fi
   7
   8 if [[ "${JOB_NAME}" == *"distribution-sanity"* ]]; then
   9     CONTROLLERMEM="4096m"
  10 else
  11     CONTROLLERMEM="3072m"
  12 fi
  13
  14 echo "Kill any controller running"
  15 # shellcheck disable=SC2009
  16 ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
  17
  18 echo "Clean Existing distribution"
  19 rm -rf "${BUNDLEFOLDER}"
  20
  21 echo "Fetch the distribution..."
  22 if  [[ -z "${BUNDLE_PATH}" ]]; then
  23     wget --progress=dot:mega  "${ACTUAL_BUNDLE_URL}"
  24 else
  25     cp "${BUNDLE_PATH}" .
  26 fi
  27
  28 echo "Extracting the new controller..."
  29 unzip -q "${BUNDLE}"
  30
  31 echo "Configuring the startup features..."
  32 FEATURESCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.apache.karaf.features.cfg"
  33 FEATURE_TEST_STRING="features-test"
  34 if [[ "$KARAF_VERSION" == "karaf3" ]]; then
  35     FEATURE_TEST_STRING="features-integration-test"
  36 fi
  37
  38 sed -ie "s%\(featuresRepositories= \|featuresRepositories = \)%featuresRepositories = mvn:org.opendaylight.integration/${FEATURE_TEST_STRING}/${BUNDLE_VERSION}/xml/features,%g" "${FEATURESCONF}"
  39
  40 if [[ -n "${REPO_URL}" ]]; then
  41    # sed below will fail if it finds space between feature repos.
  42    REPO_URL_NO_SPACE="$(echo -e "${REPO_URL}" | tr -d '[:space:]')"
  43    sed -ie "s%featuresRepositories = %featuresRepositories = ${REPO_URL_NO_SPACE},%g" "${FEATURESCONF}"
  44 fi
  45
  46 # Add actual boot features.
  47 # sed below will fail if it finds space between feature repos.
  48 FEATURES_NO_SPACE="$(echo -e "${ACTUALFEATURES}" | tr -d '[:space:]')"
  49 sed -ie "s/\(featuresBoot= \|featuresBoot = \)/featuresBoot = ${FEATURES_NO_SPACE},/g" "${FEATURESCONF}"
  50 cat "${FEATURESCONF}"
  51
  52 echo "Configuring the log..."
  53 LOGCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.ops4j.pax.logging.cfg"
  54 sed -ie 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=20MB/g' "${LOGCONF}"
  55 cat "${LOGCONF}"
  56
  57 echo "Configure max memory..."
  58 MEMCONF="${WORKSPACE}/${BUNDLEFOLDER}/bin/setenv"
  59 sed -ie "s/2048m/${CONTROLLERMEM}/g" "${MEMCONF}"
  60 cat "${MEMCONF}"
  61
  62 echo "Listing all open ports on controller system"
  63 netstat -pnatu
  64
  65 if [ "$JDKVERSION" == 'openjdk11' ]; then
  66     echo "Preparing for JRE Version 11"
  67     JAVA_HOME="/opt/jdk-11"
  68 elif [ "${JDKVERSION}" == 'openjdk8' ]; then
  69     echo "Setting the JRE Version to 8"
  70     # dynamic_verify does not allow sudo, JAVA_HOME should be enough for karaf start.
  71     # sudo /usr/sbin/alternatives --set java /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.60-2.b27.el7_1.x86_64/jre/bin/java
  72     export JAVA_HOME='/usr/lib/jvm/java-1.8.0'
  73 elif [ "${JDKVERSION}" == 'openjdk7' ]; then
  74     echo "Setting the JRE Version to 7"
  75     # dynamic_verify does not allow sudo, JAVA_HOME should be enough for karaf start.
  76     # sudo /usr/sbin/alternatives --set java /usr/lib/jvm/java-1.7.0-openjdk-1.7.0.85-2.6.1.2.el7_1.x86_64/jre/bin/java
  77     export JAVA_HOME='/usr/lib/jvm/java-1.7.0'
  78 fi
  79 readlink -e "${JAVA_HOME}/bin/java"
  80 echo "JDK Version should be overriden by JAVA_HOME"
  81 java -version
  82
  83 echo "Redirecting karaf console output to karaf_console.log"
  84 export KARAF_REDIRECT="${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
  85 mkdir -p "${WORKSPACE}/${BUNDLEFOLDER}/data/log"
  86
  87 echo "Starting controller..."
  88 "${WORKSPACE}/${BUNDLEFOLDER}/bin/start"
  89
  90 function dump_log_and_exit {
  91     echo "Dumping first 500K bytes of karaf log..."
  92     head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
  93     echo "Dumping last 500K bytes of karaf log..."
  94     tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
  95     cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
  96     cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
  97     exit 1
  98 }
  99
 100 echo "Waiting up to 6 minutes for controller to come up, checking every 5 seconds..."
 101 COUNT="0"
 102 while true; do
 103     COUNT=$(( COUNT + 5 ))
 104     sleep 5
 105     echo "already waited ${COUNT} seconds..."
 106     if grep --quiet 'org.opendaylight.infrautils.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
 107         echo "Controller is UP"
 108         break
 109     elif (( "${COUNT}" >= "360" )); then
 110         echo "Timeout Controller DOWN"
 111         dump_log_and_exit
 112     fi
 113 done
 114
 115 # echo "Checking OSGi bundles..."
 116 # sshpass seems to fail with new karaf version
 117 # sshpass -p karaf ${WORKSPACE}/${BUNDLEFOLDER}/bin/client -u karaf 'bundle:list'
 118
 119 echo "Listing all open ports on controller system"
 120 netstat -pnatu
 121
 122 function exit_on_log_file_message {
 123     echo "looking for \"$1\" in karaf.log file"
 124     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
 125         echo ABORTING: found "$1"
 126         dump_log_and_exit
 127     fi
 128
 129     echo "looking for \"$1\" in karaf_console.log file"
 130     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then
 131         echo ABORTING: found "$1"
 132         dump_log_and_exit
 133     fi
 134 }
 135
 136 exit_on_log_file_message 'Error installing boot feature repository'
 137 exit_on_log_file_message 'BindException: Address already in use'
 138 exit_on_log_file_message 'server is unhealthy'
 139
 140 echo "Fetching Karaf logs"
 141 # TODO: Move instead of copy? Gzip?
 142 cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
 143 cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
 144
 145 echo "Kill controller"
 146 # shellcheck disable=SC2009
 147 ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
 148
 149 echo "Bug 4628: Detecting misplaced config files"
 150 pushd "${WORKSPACE}/${BUNDLEFOLDER}" || exit
 151 XMLS_FOUND="$(echo -- *.xml)"
 152 popd || exit
 153 if [ "$XMLS_FOUND" != "*.xml" ]; then
 154     echo "Bug 4628 confirmed."
 155     ## TODO: Uncomment the following when ODL is fixed, to guard against regression.
 156     # exit 1
 157 else
 158     echo "Bug 4628 not detected."
 159 fi
 160
 161 # vim: ts=4 sw=4 sts=4 et ft=sh :