X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=jjb%2Fintegration%2Finclude-raw-integration-deploy-controller-run-test.sh;h=78bdec2fed423388c2bc16e9e6f3f43e51b043a7;hb=4cd27bab641230c18976d5eba338be3a416255ac;hp=73d8233d764c61cee38e5b1606fb33d3baba1bc1;hpb=3b124d322fcd7d9068ee7ea804aeefb94f85eff9;p=releng%2Fbuilder.git

diff --git a/jjb/integration/include-raw-integration-deploy-controller-run-test.sh b/jjb/integration/include-raw-integration-deploy-controller-run-test.sh
index 73d8233d7..78bdec2fe 100644
--- a/jjb/integration/include-raw-integration-deploy-controller-run-test.sh
+++ b/jjb/integration/include-raw-integration-deploy-controller-run-test.sh
@@ -1,67 +1,87 @@
-# Create a script to run controller inside a dynamic jenkins slave
+#@IgnoreInspection BashAddShebang
+# Activate robotframework virtualenv
+# ${ROBOT_VENV} comes from the include-raw-integration-install-robotframework.sh
+# script.
+source ${ROBOT_VENV}/bin/activate
+
 CONTROLLERMEM="2048m"
-DISTRIBUTION="karaf"
 
 if [ ${CONTROLLERSCOPE} == 'all' ]; then
-    CONTROLLERFEATURES="odl-integration-compatible-with-all,${CONTROLLERFEATURES}"
+    ACTUALFEATURES="odl-integration-compatible-with-all,${CONTROLLERFEATURES}"
     CONTROLLERMEM="3072m"
-fi
-
-NEXUSURL_PREFIX=${ODLNEXUSPROXY:-https://nexus.opendaylight.org}
-
-if [ ${BUNDLEURL} == 'last' ]; then
-    NEXUSPATH="${NEXUSURL_PREFIX}/content/repositories/opendaylight.snapshot/org/opendaylight/integration/distribution-${DISTRIBUTION}"
-    # Extract the BUNDLEVERSION from the pom.xml
-    BUNDLEVERSION=`xpath pom.xml '/project/version/text()' 2> /dev/null`
-    echo "Bundle version is ${BUNDLEVERSION}"
-    # Acquire the timestamp information from maven-metadata.xml
-    wget ${NEXUSPATH}/${BUNDLEVERSION}/maven-metadata.xml
-    TIMESTAMP=`xpath maven-metadata.xml "//snapshotVersion[extension='zip'][1]/value/text()" 2>/dev/null`
-    echo "Nexus timestamp is ${TIMESTAMP}"
-    BUNDLEFOLDER="distribution-${DISTRIBUTION}-${BUNDLEVERSION}"
-    BUNDLE="distribution-${DISTRIBUTION}-${TIMESTAMP}.zip"
-    BUNDLEURL="${NEXUSPATH}/${BUNDLEVERSION}/${BUNDLE}"
+    COOLDOWN_PERIOD="180"
 else
-    BUNDLE="${BUNDLEURL##*/}"
-    BUNDLEVERSION="$(basename $(dirname $BUNDLEURL))"
-    BUNDLEFOLDER="distribution-${DISTRIBUTION}-${BUNDLEVERSION}"
+    ACTUALFEATURES="${CONTROLLERFEATURES}"
+    COOLDOWN_PERIOD="60"
+fi
+# Some versions of jenkins job builder result in feature list containing spaces
+# and ending in newline. Remove all that.
+ACTUALFEATURES=`echo "${ACTUALFEATURES}" | tr -d '\n \r'`
+
+if [ -f ${WORKSPACE}/test/csit/scriptplans/${TESTPLAN} ]; then
+    echo "scriptplan exists!!!"
+    echo "Changing the scriptplan path..."
+    cat ${WORKSPACE}/test/csit/scriptplans/${TESTPLAN} | sed "s:integration:${WORKSPACE}:" > scriptplan.txt
+    cat scriptplan.txt
+    for line in $( egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' scriptplan.txt ); do
+        echo "Executing ${line}..."
+        source ${line}
+    done
 fi
-
-echo "Distribution bundle URL is ${BUNDLEURL}"
-echo "Distribution bundle is ${BUNDLE}"
-echo "Distribution folder is ${BUNDLEFOLDER}"
 
 cat > ${WORKSPACE}/controller-script.sh <<EOF
-echo "Downloading the distribution..."
+
+echo "Changing to /tmp"
 cd /tmp
-wget --no-verbose '${BUNDLEURL}'
+
+echo "Downloading the distribution..."
+wget --no-verbose '${ACTUALBUNDLEURL}'
 
 echo "Extracting the new controller..."
 unzip -q ${BUNDLE}
 
 echo "Configuring the startup features..."
-cd ${BUNDLEFOLDER}/etc
-CFG=org.apache.karaf.features.cfg
-cp \${CFG} \${CFG}.bak
-cat \${CFG}.bak | sed "s/^featuresBoot=.*/featuresBoot=config,standard,region,package,kar,ssh,management,${CONTROLLERFEATURES}/" > \${CFG}.1
-cat \${CFG}.1 | sed "s%mvn:org.opendaylight.integration/features-integration-index/${BUNDLEVERSION}/xml/features%mvn:org.opendaylight.integration/features-integration-index/${BUNDLEVERSION}/xml/features,mvn:org.opendaylight.integration/features-integration-test/${BUNDLEVERSION}/xml/features%" > \${CFG}
-cat \${CFG}
+FEATURESCONF=/tmp/${BUNDLEFOLDER}/etc/org.apache.karaf.features.cfg
+CUSTOMPROP=/tmp/${BUNDLEFOLDER}/etc/custom.properties
+sed -ie "s/featuresBoot=.*/featuresBoot=config,standard,region,package,kar,ssh,management,${ACTUALFEATURES}/g" \${FEATURESCONF}
+sed -ie "s%mvn:org.opendaylight.integration/features-integration-index/${BUNDLEVERSION}/xml/features%mvn:org.opendaylight.integration/features-integration-index/${BUNDLEVERSION}/xml/features,mvn:org.opendaylight.integration/features-integration-test/${BUNDLEVERSION}/xml/features%g" \${FEATURESCONF}
+cat \${FEATURESCONF}
+
+if [ "${ODL_ENABLE_L3_FWD}" == "yes" ]; then
+  echo "ovsdb.l3.fwd.enabled=yes" >> \${CUSTOMPROP}
+fi
+cat \${CUSTOMPROP}
 
 echo "Configuring the log..."
-LOG=org.ops4j.pax.logging.cfg
-cp \${LOG} \${LOG}.bak
-cat \${LOG}.bak | sed 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=20MB/' > \${LOG}
-cat \${LOG}
-
-echo "Configure max memory..."
-MEM=setenv
-cd ../bin
-cp \${MEM} \${MEM}.bak
-cat \${MEM}.bak | sed 's/JAVA_MAX_MEM="2048m"/JAVA_MAX_MEM="${CONTROLLERMEM}"/' > \${MEM}
-cat \${MEM}
+LOGCONF=/tmp/${BUNDLEFOLDER}/etc/org.ops4j.pax.logging.cfg
+sed -ie 's/log4j.appender.out.maxBackupIndex=10/log4j.appender.out.maxBackupIndex=1/g' \${LOGCONF}
+# FIXME: Make log size limit configurable from build parameter.
+sed -ie 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=100GB/g' \${LOGCONF}
+cat \${LOGCONF}
+
+echo "Configure java home and max memory..."
+MEMCONF=/tmp/${BUNDLEFOLDER}/bin/setenv
+sed -ie 's%^# export JAVA_HOME%export JAVA_HOME="\${JAVA_HOME:-${JAVA_HOME}}"%g' \${MEMCONF}
+sed -ie 's/JAVA_MAX_MEM="2048m"/JAVA_MAX_MEM="${CONTROLLERMEM}"/g' \${MEMCONF}
+cat \${MEMCONF}
+
+echo "Listing all open ports on controller system..."
+netstat -natu
+
+echo "Set Java version"
+sudo /usr/sbin/alternatives --install /usr/bin/java java ${JAVA_HOME}/bin/java 1
+sudo /usr/sbin/alternatives --set java ${JAVA_HOME}/bin/java
+echo "JDK default version ..."
+java -version
+
+echo "Set JAVA_HOME"
+export JAVA_HOME="$JAVA_HOME"
+# Did you know that in HERE documents, single quote is an ordinary character, but backticks are still executing?
+JAVA_RESOLVED=\`readlink -e "\${JAVA_HOME}/bin/java"\`
+echo "Java binary pointed at by JAVA_HOME: \${JAVA_RESOLVED}"
 
 echo "Starting controller..."
-./start &
+/tmp/${BUNDLEFOLDER}/bin/start
 
 echo "Waiting for controller to come up..."
 COUNT="0"
@@ -73,9 +93,12 @@ while true; do
         break
     elif (( "\$COUNT" > "600" )); then
         echo Timeout Controller DOWN
-        echo "Dumping Karaf log..."
-        cd ../data/log
-        cat karaf.log
+        echo "Dumping first 500K bytes of karaf log..."
+        head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
+        echo "Dumping last 500K bytes of karaf log..."
+        tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
+        echo "Listing all open ports on controller system"
+        netstat -natu
         exit 1
     else
         COUNT=\$(( \${COUNT} + 5 ))
@@ -84,28 +107,68 @@ while true; do
     fi
 done
 
-echo "Cool down for 1 min :)..."
-sleep 60
+echo "Cool down for ${COOLDOWN_PERIOD} seconds :)..."
+sleep ${COOLDOWN_PERIOD}
 
-echo "Checking OSGi bundles..."
-sshpass -p karaf ./client -u karaf 'bundle:list'
+echo "Listing all open ports on controller system..."
+netstat -natu
+
+function exit_on_log_file_message {
+    echo "looking for \"\$1\" in log file"
+    if grep --quiet "\$1" /tmp/${BUNDLEFOLDER}/data/log/karaf.log; then
+        echo ABORTING: found "\$1"
+        echo "Dumping first 500K bytes of karaf log..."
+        head --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
+        echo "Dumping last 500K bytes of karaf log..."
+        tail --bytes=500K "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
+        exit 1
+    fi
+}
+
+exit_on_log_file_message 'BindException: Address already in use'
+exit_on_log_file_message 'server is unhealthy'
 
 EOF
 
-scp ${WORKSPACE}/controller-script.sh ${CONTROLLER0}:/tmp
-ssh ${CONTROLLER0} 'bash /tmp/controller-script.sh'
+scp ${WORKSPACE}/controller-script.sh ${ODL_SYSTEM_IP}:/tmp
+ssh ${ODL_SYSTEM_IP} 'bash /tmp/controller-script.sh'
+
+if [ ${NUM_OPENSTACK_SYSTEM} -gt 0 ]; then
+   echo "Exiting without running tests to deploy openstack for testing"
+   exit
+fi
+
+echo "Locating test plan to use..."
+testplan_filepath="${WORKSPACE}/test/csit/testplans/${STREAMTESTPLAN}"
+if [ ! -f "${testplan_filepath}" ]; then
+    testplan_filepath="${WORKSPACE}/test/csit/testplans/${TESTPLAN}"
+fi
 
 echo "Changing the testplan path..."
-cat ${WORKSPACE}/test/csit/testplans/${TESTPLAN} | sed "s:integration:${WORKSPACE}:" > testplan.txt
+cat "${testplan_filepath}" | sed "s:integration:${WORKSPACE}:" > testplan.txt
 cat testplan.txt
-
 SUITES=$( egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' testplan.txt | tr '\012' ' ' )
 
 echo "Starting Robot test suites ${SUITES} ..."
-pybot -N ${TESTPLAN} -c critical -e exclude --loglevel TRACE -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp -v CONTROLLER:${CONTROLLER0} -v MININET:${MININET0} -v MININET_USER:${USER} -v USER_HOME:${HOME} -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} ${TESTOPTIONS} ${SUITES}
-
-echo "Fetching Karaf log"
-scp ${CONTROLLER0}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log .
+pybot -N ${TESTPLAN} -c critical -e exclude -v BUNDLEFOLDER:${BUNDLEFOLDER} -v WORKSPACE:/tmp \
+-v JAVA_HOME:${JAVA_HOME} -v BUNDLE_URL:${ACTUALBUNDLEURL} -v NEXUSURL_PREFIX:${NEXUSURL_PREFIX} \
+-v CONTROLLER:${ODL_SYSTEM_IP} -v ODL_SYSTEM_IP:${ODL_SYSTEM_IP} -v CONTROLLER_USER:${USER} -v ODL_SYSTEM_USER:${USER} \
+-v TOOLS_SYSTEM_IP:${TOOLS_SYSTEM_IP} -v TOOLS_SYSTEM_2_IP:${TOOLS_SYSTEM_2_IP} -v TOOLS_SYSTEM_3_IP:${TOOLS_SYSTEM_3_IP} \
+-v TOOLS_SYSTEM_4_IP:${TOOLS_SYSTEM_4_IP} -v TOOLS_SYSTEM_5_IP:${TOOLS_SYSTEM_5_IP} -v TOOLS_SYSTEM_6_IP:${TOOLS_SYSTEM_6_IP} \
+-v TOOLS_SYSTEM_USER:${USER} -v JDKVERSION:${JDKVERSION} -v ODL_STREAM:${DISTROSTREAM} \
+-v MININET:${TOOLS_SYSTEM_IP} -v MININET1:${TOOLS_SYSTEM_2_IP} -v MININET2:${TOOLS_SYSTEM_3_IP} \
+-v MININET3:${TOOLS_SYSTEM_4_IP} -v MININET4:${TOOLS_SYSTEM_5_IP} -v MININET5:${TOOLS_SYSTEM_6_IP} \
+-v MININET_USER:${USER} -v USER_HOME:${HOME} ${TESTOPTIONS} ${SUITES} || true
+# FIXME: Sort (at least -v) options alphabetically.
+
+echo "Killing ODL and fetching Karaf log..."
+set +e  # We do not want to create red dot just because something went wrong while fetching logs.
+ssh "${ODL_SYSTEM_IP}" tail --bytes=1M "/tmp/${BUNDLEFOLDER}/data/log/karaf.log" > "karaf.log"
+ssh "${ODL_SYSTEM_IP}" bash -c 'ps axf | grep karaf | grep -v grep | awk '"'"'{print "kill -9 " $1}'"'"' | sh'
+sleep 5
+ssh "${ODL_SYSTEM_IP}" xz -9ekvv "/tmp/${BUNDLEFOLDER}/data/log/karaf.log"
+scp "${ODL_SYSTEM_IP}:/tmp/${BUNDLEFOLDER}/data/log/karaf.log.xz" .
+true  # perhaps Jenkins is testing last exit code
 
 # vim: ts=4 sw=4 sts=4 et ft=sh :