Add controller ready detect to int/dist jobs

These are distribution-check and distribution-deploy jobs.

Change-Id: I7e57b210ee21b4ba29196d47e958da3771b5455a
Signed-off-by: Luis Gomez <ecelgp@gmail.com>

diff --git a/jjb/integration/distribution/distribution-check-bootup.sh b/jjb/integration/distribution/distribution-check-bootup.sh
index 2a333dea772f553e762e41d85cdced555f814f81..35076e1fc09ad550a9f0feae86a256a093cd2522 100644 (file)
--- a/jjb/integration/distribution/distribution-check-bootup.sh
+++ b/jjb/integration/distribution/distribution-check-bootup.sh
@@ -73,42 +73,50 @@ ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
 # No need for verbose printing during repeating operations.
 set +x
 
-echo "Waiting for controller to come up..."
-COUNT=0
-while true; do
-    RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)"
-    echo "${RESP}"
-    if [[ "${RESP}" == *"200"* ]]; then
-        echo Controller is UP
-        break
-    elif (( "${COUNT}" > 600 )); then
-        echo Timeout Controller DOWN
-        echo "Dumping Karaf log..."
-        cat "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Listing all open ports on controller system"
-        netstat -pnatu
-        exit 1
-    else
-        COUNT=$(( ${COUNT} + 1 ))
-        sleep 1
-        if [[ $(($COUNT % 5)) == 0 ]]; then
-            echo already waited ${COUNT} seconds...
-        fi
-    fi
-done
+function dump_log_and_exit {
+    echo "Dumping first 500K bytes of karaf log..."
+    head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
+    echo "Dumping last 500K bytes of karaf log..."
+    tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
+    cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
+    cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
+    exit 1
+}
 
-echo "loading many features at once.  Need to allow time for problems to show up in logs.  cool down for 5 min ..."
-COUNT="300"
-while true; do
-    if (( "${COUNT}" <= "0" )); then
-        break
-    fi
-    echo "${COUNT} seconds yet to wait..."
-    sleep 10
-    COUNT="$(( ${COUNT} - 10 ))"
-done
+echo "Waiting up to 5 minutes for controller to come up, checking every 5 seconds..."
+if [ "${DISTROSTREAM}" == "carbon" ] || [ "${DISTROSTREAM}" == "nitrogen" ]; then
+    # Only oxygen and above have the infrautils.ready feature, so using REST API to determine if the controller is ready.
+    COUNT="0"
+    while true; do
+        COUNT=$(( ${COUNT} + 5 ))
+        sleep 5
+        echo "already waited ${COUNT} seconds..."
+        RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)"
+        echo ${RESP}
+        if [[ ${RESP} == *"200"* ]]; then
+            echo "Controller is UP"
+            break
+        elif (( "${COUNT}" > "300" )); then
+            echo "Timeout Controller DOWN"
+            dump_log_and_exit
+        fi
+    done
+else
+    COUNT="0"
+    while true; do
+        COUNT=$(( ${COUNT} + 5 ))
+        sleep 5
+        echo "already waited ${COUNT} seconds..."
+        if grep --quiet 'org.opendaylight.infrautils.ready-impl.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
+            echo "Controller is UP"
+            break
+        elif (( "${COUNT}" > "300" )); then
+            echo "Timeout Controller DOWN"
+            dump_log_and_exit
+        fi
+    done
+fi
 
-# End of repeating operations, enable verbose printing.
 set -x
 
 # echo "Checking OSGi bundles..."
@@ -122,25 +130,13 @@ function exit_on_log_file_message {
     echo "looking for \"$1\" in karaf.log file"
     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
         echo ABORTING: found "$1"
-        echo "Dumping first 500K bytes of karaf log..."
-        head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Dumping last 500K bytes of karaf log..."
-        tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
-        exit 1
+        dump_log_and_exit
     fi
 
     echo "looking for \"$1\" in karaf_console.log file"
     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then
         echo ABORTING: found "$1"
-        echo "Dumping first 500K bytes of karaf log..."
-        head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
-        echo "Dumping last 500K bytes of karaf log..."
-        tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
-        exit 1
+        dump_log_and_exit
     fi
 }
 

diff --git a/jjb/integration/distribution/distribution-deploy-verify.sh b/jjb/integration/distribution/distribution-deploy-verify.sh
index c81ac6db6734fd7472c28236fe801a4ead40a633..ba28c7f6474f5b0803d028a30156257aed4f27b2 100644 (file)
--- a/jjb/integration/distribution/distribution-deploy-verify.sh
+++ b/jjb/integration/distribution/distribution-deploy-verify.sh
@@ -61,34 +61,51 @@ mkdir -p ${WORKSPACE}/${BUNDLEFOLDER}/data/log
 echo "Starting controller..."
 ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
 
-echo "Waiting for controller to come up..."
-COUNT=0
-while true; do
-    RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)"
-    echo "${RESP}"
-    if [[ "${RESP}" == *"200"* ]]; then
-        echo Controller is UP
-        break
-    elif (( "${COUNT}" > 600 )); then
-        echo Timeout Controller DOWN
-        echo "Dumping Karaf log..."
-        cat "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Listing all open ports on controller system"
-        netstat -pnatu
-        exit 1
-    else
-        COUNT=$(( ${COUNT} + 1 ))
-        sleep 1
-        if [[ $(($COUNT % 5)) == 0 ]]; then
-            echo already waited ${COUNT} seconds...
-        fi
-    fi
-done
+function dump_log_and_exit {
+    echo "Dumping first 500K bytes of karaf log..."
+    head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
+    echo "Dumping last 500K bytes of karaf log..."
+    tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
+    cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
+    cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
+    exit 1
+}
 
-echo "loading many features at once.  Need to allow time for problems to show up in logs.  cool down for 5 min ..."
-sleep 300
+echo "Waiting up to 5 minutes for controller to come up, checking every 5 seconds..."
+if [ "${DISTROSTREAM}" == "carbon" ] || [ "${DISTROSTREAM}" == "nitrogen" ]; then
+    # Only oxygen and above have the infrautils.ready feature, so using REST API to determine if the controller is ready.
+    COUNT="0"
+    while true; do
+        COUNT=$(( ${COUNT} + 5 ))
+        sleep 5
+        echo "already waited ${COUNT} seconds..."
+        RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)"
+        echo ${RESP}
+        if [[ ${RESP} == *"200"* ]]; then
+            echo "Controller is UP"
+            break
+        elif (( "${COUNT}" > "300" )); then
+            echo "Timeout Controller DOWN"
+            dump_log_and_exit
+        fi
+    done
+else
+    COUNT="0"
+    while true; do
+        COUNT=$(( ${COUNT} + 5 ))
+        sleep 5
+        echo "already waited ${COUNT} seconds..."
+        if grep --quiet 'org.opendaylight.infrautils.ready-impl.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
+            echo "Controller is UP"
+            break
+        elif (( "${COUNT}" > "300" )); then
+            echo "Timeout Controller DOWN"
+            dump_log_and_exit
+        fi
+    done
+fi
 
-echo "Checking OSGi bundles..."
+# echo "Checking OSGi bundles..."
 # sshpass seems to fail with new karaf version
 # sshpass -p karaf ${WORKSPACE}/${BUNDLEFOLDER}/bin/client -u karaf 'bundle:list'
 
@@ -99,25 +116,13 @@ function exit_on_log_file_message {
     echo "looking for \"$1\" in karaf.log file"
     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then
         echo ABORTING: found "$1"
-        echo "Dumping first 500K bytes of karaf log..."
-        head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
-        echo "Dumping last 500K bytes of karaf log..."
-        tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
-        exit 1
+        dump_log_and_exit
     fi
 
     echo "looking for \"$1\" in karaf_console.log file"
     if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then
         echo ABORTING: found "$1"
-        echo "Dumping first 500K bytes of karaf log..."
-        head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
-        echo "Dumping last 500K bytes of karaf log..."
-        tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" .
-        cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" .
-        exit 1
+        dump_log_and_exit
     fi
 }
 

diff --git a/jjb/integration/distribution/distribution-jobs.yaml b/jjb/integration/distribution/distribution-jobs.yaml
index 1ffd0ea7d9d4680920ff929e00d312b5aa9b1f91..94e9f13ccda0d93b95ecd7dd0a9eb4ae6fced6c1 100644 (file)
--- a/jjb/integration/distribution/distribution-jobs.yaml
+++ b/jjb/integration/distribution/distribution-jobs.yaml
@@ -218,6 +218,8 @@
           branch: '{branch}'
           refspec: 'refs/heads/{branch}'
           artifacts: '{archive-artifacts}'
+      - integration-distribution-stream:
+          stream: '{stream}'
       - distribution-karaf-version:
           karaf-version: '{karaf-version}'
       - maven-exec:
@@ -296,6 +298,8 @@
           branch: '{branch}'
           refspec: 'refs/heads/{branch}'
           artifacts: '{archive-artifacts}'
+      - integration-distribution-stream:
+          stream: '{stream}'
       - integration-distribution-branch:
           branch: '{branch}'
       - integration-bundle-url:

diff --git a/jjb/integration/distribution/distribution-templates.yaml b/jjb/integration/distribution/distribution-templates.yaml
index e7c0746e20a258cc8fb49fbdc7066239f14e8445..4453612348a80996a87379cb8d6abfa2e8d92443 100644 (file)
--- a/jjb/integration/distribution/distribution-templates.yaml
+++ b/jjb/integration/distribution/distribution-templates.yaml
@@ -37,6 +37,8 @@
           branch: '{branch}'
           refspec: 'refs/heads/{branch}'
           artifacts: '{archive-artifacts}'
+      - integration-distribution-stream:
+          stream: '{stream}'
       - integration-distribution-git-url:
           git-url: '{git-url}'
       - distribution-karaf-version: