jjb/integration/distribution/distribution-check-bootup.sh

   1
   2 CONTROLLERMEM="3072m"
   3 ACTUALFEATURES="odl-integration-all"
   4 BUNDLEVERSION="$(xpath distribution/pom.xml '/project/version/text()' 2> /dev/null)"
   5 BUNDLEFOLDER="${KARAF_ARTIFACT}-${BUNDLEVERSION}"
   6 BUNDLE="${BUNDLEFOLDER}.zip"
   7 BUNDLE_PATH="/tmp/r/org/opendaylight/integration/${KARAF_ARTIFACT}/${BUNDLEVERSION}/${BUNDLE}"
   8
   9 echo "Kill any controller running"
  10 ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
  11
  12 echo "Clean Existing distribution"
  13 rm -rf ${BUNDLEFOLDER}
  14
  15 echo "Copying the distribution..."
  16 cp "${BUNDLE_PATH}" .
  17
  18 echo "Extracting the new controller..."
  19 unzip -q "${BUNDLE}"
  20
  21 echo "Configuring the startup features..."
  22 FEATURESCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.apache.karaf.features.cfg"
  23 FEATURE_TEST_STRING="features-integration-test"
  24 if [[ "$KARAF_VERSION" == "karaf4" ]]; then
  25     FEATURE_TEST_STRING="features-test"
  26 fi
  27
  28 sed -ie "s%\(featuresRepositories=\|featuresRepositories =\)%featuresRepositories = mvn:org.opendaylight.integration/${FEATURE_TEST_STRING}/${BUNDLEVERSION}/xml/features,%g" ${FEATURESCONF}
  29
  30 # Add actual boot features.
  31 sed -ie "s/\(featuresBoot=\|featuresBoot =\)/featuresBoot = ${ACTUALFEATURES},/g" "${FEATURESCONF}"
  32 cat "${FEATURESCONF}"
  33
  34 echo "Configuring the log..."
  35 LOGCONF="${WORKSPACE}/${BUNDLEFOLDER}/etc/org.ops4j.pax.logging.cfg"
  36 sed -ie 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=20MB/g' "${LOGCONF}"
  37 cat "${LOGCONF}"
  38
  39 echo "Configure max memory..."
  40 MEMCONF="${WORKSPACE}/${BUNDLEFOLDER}/bin/setenv"
  41 sed -ie "s/2048m/${CONTROLLERMEM}/g" "${MEMCONF}"
  42 cat "${MEMCONF}"
  43
  44 echo "Listing all open ports on controller system"
  45 netstat -pnatu
  46
  47 echo "redirected karaf console output to karaf_console.log"
  48 export KARAF_REDIRECT="${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
  49 mkdir -p ${WORKSPACE}/${BUNDLEFOLDER}/data/log
  50
  51 if [ "${JDKVERSION}" == 'openjdk8' ]; then
  52     echo "Setting the JRE Version to 8"
  53     # dynamic_verify does not allow sudo, JAVA_HOME should be enough for karaf start.
  54     # sudo /usr/sbin/alternatives --set java /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.60-2.b27.el7_1.x86_64/jre/bin/java
  55     export JAVA_HOME='/usr/lib/jvm/java-1.8.0'
  56 elif [ "${JDKVERSION}" == 'openjdk7' ]; then
  57     echo "Setting the JRE Version to 7"
  58     # dynamic_verify does not allow sudo, JAVA_HOME should be enough for karaf start.
  59     # sudo /usr/sbin/alternatives --set java /usr/lib/jvm/java-1.7.0-openjdk-1.7.0.85-2.6.1.2.el7_1.x86_64/jre/bin/java
  60     export JAVA_HOME='/usr/lib/jvm/java-1.7.0'
  61 fi
  62 readlink -e "${JAVA_HOME}/bin/java"
  63 echo "JDK Version should be overriden by JAVA_HOME"
  64 java -version
  65
  66 echo "Redirecting karaf console output to karaf_console.log"
  67 export KARAF_REDIRECT="${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
  68 mkdir -p ${WORKSPACE}/${BUNDLEFOLDER}/data/log
  69
  70 echo "Starting controller..."
  71 ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
  72
  73 # No need for verbose printing during repeating operations.
  74 set +x
  75
  76 function dump_log_and_exit {
  77     echo "Dumping first 500K bytes of karaf log..."
  78     head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
  79     echo "Dumping last 500K bytes of karaf log..."
  80     tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
  81     cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
  82     cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
  83     exit 1
  84 }
  85
  86 echo "Waiting up to 5 minutes for controller to come up, checking every 5 seconds..."
  87 if [ "${DISTROSTREAM}" == "carbon" ] || [ "${DISTROSTREAM}" == "nitrogen" ]; then
  88     # Only oxygen and above have the infrautils.ready feature, so using REST API to determine if the controller is ready.
  89     COUNT="0"
  90     while true; do
  91         COUNT=$(( ${COUNT} + 5 ))
  92         sleep 5
  93         echo "already waited ${COUNT} seconds..."
  94         RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)"
  95         echo ${RESP}
  96         if [[ ${RESP} == *"200"* ]]; then
  97             echo "Controller is UP"
  98             break
  99         elif (( "${COUNT}" > "300" )); then
 100             echo "Timeout Controller DOWN"
 101             dump_log_and_exit
 102         fi
 103     done
 104 else
 105     COUNT="0"
 106     while true; do
 107         COUNT=$(( ${COUNT} + 5 ))
 108         sleep 5
 109         echo "already waited ${COUNT} seconds..."
 110         if grep --quiet 'org.opendaylight.infrautils.ready-impl.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
 111             echo "Controller is UP"
 112             break
 113         elif (( "${COUNT}" > "300" )); then
 114             echo "Timeout Controller DOWN"
 115             dump_log_and_exit
 116         fi
 117     done
 118 fi
 119
 120 set -x
 121
 122 # echo "Checking OSGi bundles..."
 123 # sshpass seems to fail with new karaf version
 124 # sshpass -p karaf ${WORKSPACE}/${BUNDLEFOLDER}/bin/client -u karaf 'bundle:list'
 125
 126 echo "Listing all open ports on controller system"
 127 netstat -pnatu
 128
 129 function exit_on_log_file_message {
 130     echo "looking for \"$1\" in karaf.log file"
 131     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
 132         echo ABORTING: found "$1"
 133         dump_log_and_exit
 134     fi
 135
 136     echo "looking for \"$1\" in karaf_console.log file"
 137     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then
 138         echo ABORTING: found "$1"
 139         dump_log_and_exit
 140     fi
 141 }
 142
 143 exit_on_log_file_message 'BindException: Address already in use'
 144 exit_on_log_file_message 'server is unhealthy'
 145
 146 echo "Fetching Karaf logs"
 147 # TODO: Move instead of copy? Gzip?
 148 cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
 149 cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
 150
 151 echo "Kill controller"
 152 ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
 153
 154 echo "Bug 4628: Detecting misplaced config files"
 155 pushd "${WORKSPACE}/${BUNDLEFOLDER}" || exit
 156 XMLS_FOUND="$(echo *.xml)"
 157 popd || exit
 158 if [ "$XMLS_FOUND" != "*.xml" ]; then
 159     echo "Bug 4628 confirmed."
 160     ## TODO: Uncomment the following when ODL is fixed, to guard against regression.
 161     # exit 1
 162 else
 163     echo "Bug 4628 not detected."
 164 fi
 165
 166 # vim: ts=4 sw=4 sts=4 et ft=sh :