From: Luis Gomez <ecelgp@gmail.com>
Date: Thu, 11 Feb 2016 09:33:38 +0000 (-0800)
Subject: Change distribution-check job to do verification in same job
X-Git-Tag: release/beryllium~21
X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=releng%2Fbuilder.git;a=commitdiff_plain;h=086b2dabf46777a37c7bb0e927044b529bda1041

Change distribution-check job to do verification in same job

Change-Id: I816a6b21ab4d7bdb3a0d44eefbc22a4b85eedd25
Signed-off-by: Luis Gomez <ecelgp@gmail.com>
---

diff --git a/jjb-templates/distribution-check.yaml b/jjb-templates/distribution-check.yaml
index c75407f50..4a77f22c9 100644
--- a/jjb-templates/distribution-check.yaml
+++ b/jjb-templates/distribution-check.yaml
@@ -67,9 +67,6 @@
                 notbuilt: true
 
     prebuilders:
-        - integration-get-bundle-url
-        - inject:
-            properties-file: 'bundle.txt'
         - wipe-org-opendaylight-repo
         - maven-target:
             maven-version: '{mvn33}'
@@ -91,12 +88,12 @@
         post-step-run-condition: 'SUCCESS'
 
     postbuilders:
-        - trigger-builds:
-            - project: 'integration-distribution-deploy-{stream}'
-              block: true
-              predefined-parameters:
-                  BUNDLEURL=$BUNDLEURL
+        - integration-distribution-check
 
     publishers:
+        - archive:
+            artifacts: 'karaf.log'
+        - archive:
+            artifacts: 'karaf_console.log'
         - email-notification:
             email-prefix: '[PROJECT_SHORTNAME]'
diff --git a/jjb/integration/include-raw-integration-distribution-check.sh b/jjb/integration/include-raw-integration-distribution-check.sh
new file mode 100644
index 000000000..d0c8b002c
--- /dev/null
+++ b/jjb/integration/include-raw-integration-distribution-check.sh
@@ -0,0 +1,127 @@
+CONTROLLERMEM="3072m"
+ACTUALFEATURES="odl-integration-all"
+BUNDLEVERSION=`xpath distribution/pom.xml '/project/version/text()' 2> /dev/null`
+BUNDLEFOLDER="distribution-karaf-${BUNDLEVERSION}"
+BUNDLE="${BUNDLEFOLDER}.zip"
+BUNDLEURL="/tmp/r/org/opendaylight/integration/distribution-karaf/${BUNDLEVERSION}/${BUNDLE}"
+
+echo "Kill any controller running"
+ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
+
+echo "Clean workspace"
+rm -rf *
+
+echo "Copying the distribution..."
+cp  ${BUNDLEURL} .
+
+echo "Extracting the new controller..."
+unzip -q ${BUNDLE}
+
+echo "Configuring the startup features..."
+FEATURESCONF=${WORKSPACE}/${BUNDLEFOLDER}/etc/org.apache.karaf.features.cfg
+sed -ie "s/featuresBoot=.*/featuresBoot=config,standard,region,package,kar,ssh,management,${ACTUALFEATURES}/g" ${FEATURESCONF}
+sed -ie "s%mvn:org.opendaylight.integration/features-integration-index/${BUNDLEVERSION}/xml/features%mvn:org.opendaylight.integration/features-integration-index/${BUNDLEVERSION}/xml/features,mvn:org.opendaylight.integration/features-integration-test/${BUNDLEVERSION}/xml/features%g" ${FEATURESCONF}
+cat ${FEATURESCONF}
+
+echo "Configuring the log..."
+LOGCONF=${WORKSPACE}/${BUNDLEFOLDER}/etc/org.ops4j.pax.logging.cfg
+sed -ie 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=20MB/g' ${LOGCONF}
+cat ${LOGCONF}
+
+echo "Configure max memory..."
+MEMCONF=${WORKSPACE}/${BUNDLEFOLDER}/bin/setenv
+sed -ie "s/2048m/${CONTROLLERMEM}/g" ${MEMCONF}
+cat ${MEMCONF}
+
+echo "Listing all open ports on controller system"
+netstat -natu
+
+echo "redirected karaf console output to karaf_console.log"
+export KARAF_REDIRECT=${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log
+
+echo "JDK Version ..."
+java -version
+
+echo "Starting controller..."
+${WORKSPACE}/${BUNDLEFOLDER}/bin/start
+
+echo "Waiting for controller to come up..."
+COUNT=0
+while true; do
+    RESP="$( curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true )"
+    echo ${RESP}
+    if [[ ${RESP} == *"200"* ]]; then
+        echo Controller is UP
+        break
+    elif (( ${COUNT} > 600 )); then
+        echo Timeout Controller DOWN
+        echo "Dumping Karaf log..."
+        cat ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log
+        echo "Listing all open ports on controller system"
+        netstat -natu
+        exit 1
+    else
+        COUNT=$(( ${COUNT} + 5 ))
+        sleep 5
+        echo waiting ${COUNT} secs...
+    fi
+done
+
+echo "loading many features at once.  Need to allow time for problems to show up in logs.  cool down for 5 min ..."
+sleep 300
+
+echo "Checking OSGi bundles..."
+sshpass -p karaf ${WORKSPACE}/${BUNDLEFOLDER}/bin/client -u karaf 'bundle:list'
+
+echo "Listing all open ports on controller system"
+netstat -natu
+
+function exit_on_log_file_message {
+    echo "looking for \"$1\" in karaf.log file"
+    if grep --quiet "$1" ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log; then
+        echo ABORTING: found "$1"
+        echo "Dumping first 500K bytes of karaf log..."
+        head --bytes=500K  ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log
+        echo "Dumping last 500K bytes of karaf log..."
+        tail --bytes=500K  ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log
+        cp ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log .
+        cp ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log .
+        exit 1
+    fi
+
+    echo "looking for \"$1\" in karaf_console.log file"
+    if grep --quiet "$1" ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log; then
+        echo ABORTING: found "$1"
+        echo "Dumping first 500K bytes of karaf log..."
+        head --bytes=500K  ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log
+        echo "Dumping last 500K bytes of karaf log..."
+        tail --bytes=500K  ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log
+        cp ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log .
+        cp ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log .
+        exit 1
+    fi
+}
+
+exit_on_log_file_message 'BindException: Address already in use'
+exit_on_log_file_message 'server is unhealthy'
+
+echo "Fetching Karaf logs"
+cp ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log .
+cp ${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log .
+
+echo "Kill controller"
+ps axf | grep karaf | grep -v grep | awk '{print "kill -9 " $1}' | sh
+
+echo "Detecting misplaced config files"
+pushd "${WORKSPACE}/${BUNDLEFOLDER}"
+XMLS_FOUND=`echo *.xml`
+popd
+if [ "$XMLS_FOUND" != "*.xml" ]; then
+    echo "Bug 4628 confirmed."
+    ## TODO: Uncomment the following when ODL is fixed, to guard against regression.
+    # exit 1
+else
+    echo "Bug 4628 not detected."
+fi
+
+# vim: ts=4 sw=4 sts=4 et ft=sh :
diff --git a/jjb/integration/integration-macros.yaml b/jjb/integration/integration-macros.yaml
index 19f321b9e..9a82329ce 100644
--- a/jjb/integration/integration-macros.yaml
+++ b/jjb/integration/integration-macros.yaml
@@ -222,9 +222,20 @@
             !include-raw:
                 - include-raw-integration-get-bundle-vars.sh
 
+# Macro: integration-distribution-check
+# Operation: this macro deploys the controller with all fetures
+# Used by: {project}-distribution-check-{stream} job template
+
+- builder:
+    name: integration-distribution-check
+    builders:
+        - shell:
+            !include-raw:
+                - include-raw-integration-distribution-check.sh
+
 # Macro: integration-deploy-controller-verify
 # Operation: this macro deploys the controller with all fetures
-# Used by: integration-distrbution-deploy-{stream} job template
+# Used by: integration-distribution-deploy-{stream} job template
 
 - builder:
     name: integration-deploy-controller-verify