jjb/integration/distribution/distribution-check-bootup.sh

   1 if [[ ! -z "${CONTROLLERFEATURES}" ]]; then
   2     ACTUALFEATURES="odl-integration-all,${CONTROLLERFEATURES}"
   3 else
   4     ACTUALFEATURES="odl-integration-all"
   5 fi
   6
   7 if [[ "${JOB_NAME}" == *"distribution-sanity"* ]]; then
   8     CONTROLLERMEM="4096m"
   9 else
  10     CONTROLLERMEM="3072m"
  11 fi
  12
  13 echo "Kill any controller running"
  14 ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
  15
  16 echo "Clean Existing distribution"
  17 rm -rf ${BUNDLEFOLDER}
  18
  19 echo "Fetch the distribution..."
  20 if  [[ -z "${BUNDLE_PATH}" ]]; then
  21     wget --progress=dot:mega  "${ACTUAL_BUNDLE_URL}"
  22 else
  23     cp "${BUNDLE_PATH}" .
  24 fi
  25
  26 echo "Extracting the new controller..."
  27 unzip -q "${BUNDLE}"
  28
  29 echo "Configuring the startup features..."
  30 FEATURESCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.apache.karaf.features.cfg"
  31 FEATURE_TEST_STRING="features-test"
  32 if [[ "$KARAF_VERSION" == "karaf3" ]]; then
  33     FEATURE_TEST_STRING="features-integration-test"
  34 fi
  35
  36 sed -ie "s%\(featuresRepositories= \|featuresRepositories = \)%featuresRepositories = mvn:org.opendaylight.integration/${FEATURE_TEST_STRING}/${BUNDLE_VERSION}/xml/features,%g" ${FEATURESCONF}
  37
  38 if [[ ! -z "${REPO_URL}" ]]; then
  39    # sed below will fail if it finds space between feature repos.
  40    REPO_URL_NO_SPACE="$(echo -e "${REPO_URL}" | tr -d '[:space:]')"
  41    sed -ie "s%featuresRepositories = %featuresRepositories = ${REPO_URL_NO_SPACE},%g" ${FEATURESCONF}
  42 fi
  43
  44 # Add actual boot features.
  45 # sed below will fail if it finds space between feature repos.
  46 FEATURES_NO_SPACE="$(echo -e "${ACTUALFEATURES}" | tr -d '[:space:]')"
  47 sed -ie "s/\(featuresBoot= \|featuresBoot = \)/featuresBoot = ${FEATURES_NO_SPACE},/g" "${FEATURESCONF}"
  48 cat "${FEATURESCONF}"
  49
  50 echo "Configuring the log..."
  51 LOGCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.ops4j.pax.logging.cfg"
  52 sed -ie 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=20MB/g' "${LOGCONF}"
  53 cat "${LOGCONF}"
  54
  55 echo "Configure max memory..."
  56 MEMCONF="${WORKSPACE}/${BUNDLEFOLDER}/bin/setenv"
  57 sed -ie "s/2048m/${CONTROLLERMEM}/g" "${MEMCONF}"
  58 cat "${MEMCONF}"
  59
  60 echo "Listing all open ports on controller system"
  61 netstat -pnatu
  62
  63 if [ "$JDKVERSION" == 'openjdk11' ]; then
  64     echo "Preparing for JRE Version 11"
  65     JAVA_HOME="/opt/jdk-11"
  66 elif [ "${JDKVERSION}" == 'openjdk8' ]; then
  67     echo "Setting the JRE Version to 8"
  68     # dynamic_verify does not allow sudo, JAVA_HOME should be enough for karaf start.
  69     # sudo /usr/sbin/alternatives --set java /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.60-2.b27.el7_1.x86_64/jre/bin/java
  70     export JAVA_HOME='/usr/lib/jvm/java-1.8.0'
  71 elif [ "${JDKVERSION}" == 'openjdk7' ]; then
  72     echo "Setting the JRE Version to 7"
  73     # dynamic_verify does not allow sudo, JAVA_HOME should be enough for karaf start.
  74     # sudo /usr/sbin/alternatives --set java /usr/lib/jvm/java-1.7.0-openjdk-1.7.0.85-2.6.1.2.el7_1.x86_64/jre/bin/java
  75     export JAVA_HOME='/usr/lib/jvm/java-1.7.0'
  76 fi
  77 readlink -e "${JAVA_HOME}/bin/java"
  78 echo "JDK Version should be overriden by JAVA_HOME"
  79 java -version
  80
  81 echo "Redirecting karaf console output to karaf_console.log"
  82 export KARAF_REDIRECT="${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
  83 mkdir -p ${WORKSPACE}/${BUNDLEFOLDER}/data/log
  84
  85 echo "Starting controller..."
  86 ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
  87
  88 function dump_log_and_exit {
  89     echo "Dumping first 500K bytes of karaf log..."
  90     head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
  91     echo "Dumping last 500K bytes of karaf log..."
  92     tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
  93     cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
  94     cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
  95     exit 1
  96 }
  97
  98 echo "Waiting up to 6 minutes for controller to come up, checking every 5 seconds..."
  99 COUNT="0"
 100 while true; do
 101     COUNT=$(( ${COUNT} + 5 ))
 102     sleep 5
 103     echo "already waited ${COUNT} seconds..."
 104     if grep --quiet 'org.opendaylight.infrautils.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
 105         echo "Controller is UP"
 106         break
 107     elif (( "${COUNT}" >= "360" )); then
 108         echo "Timeout Controller DOWN"
 109         dump_log_and_exit
 110     fi
 111 done
 112
 113 # echo "Checking OSGi bundles..."
 114 # sshpass seems to fail with new karaf version
 115 # sshpass -p karaf ${WORKSPACE}/${BUNDLEFOLDER}/bin/client -u karaf 'bundle:list'
 116
 117 echo "Listing all open ports on controller system"
 118 netstat -pnatu
 119
 120 function exit_on_log_file_message {
 121     echo "looking for \"$1\" in karaf.log file"
 122     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
 123         echo ABORTING: found "$1"
 124         dump_log_and_exit
 125     fi
 126
 127     echo "looking for \"$1\" in karaf_console.log file"
 128     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then
 129         echo ABORTING: found "$1"
 130         dump_log_and_exit
 131     fi
 132 }
 133
 134 exit_on_log_file_message 'Error installing boot feature repository'
 135 exit_on_log_file_message 'BindException: Address already in use'
 136 exit_on_log_file_message 'server is unhealthy'
 137
 138 echo "Fetching Karaf logs"
 139 # TODO: Move instead of copy? Gzip?
 140 cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
 141 cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
 142
 143 echo "Kill controller"
 144 ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
 145
 146 echo "Bug 4628: Detecting misplaced config files"
 147 pushd "${WORKSPACE}/${BUNDLEFOLDER}" || exit
 148 XMLS_FOUND="$(echo *.xml)"
 149 popd || exit
 150 if [ "$XMLS_FOUND" != "*.xml" ]; then
 151     echo "Bug 4628 confirmed."
 152     ## TODO: Uncomment the following when ODL is fixed, to guard against regression.
 153     # exit 1
 154 else
 155     echo "Bug 4628 not detected."
 156 fi
 157
 158 # vim: ts=4 sw=4 sts=4 et ft=sh :