From ffb931574d302145620add4c8d4f54fab9c9aede Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Mon, 16 Apr 2018 16:09:23 -0700 Subject: [PATCH] Add controller ready detect to int/dist jobs These are distribution-check and distribution-deploy jobs. Change-Id: I7e57b210ee21b4ba29196d47e958da3771b5455a Signed-off-by: Luis Gomez --- .../distribution/distribution-check-bootup.sh | 92 +++++++++---------- .../distribution-deploy-verify.sh | 85 +++++++++-------- .../distribution/distribution-jobs.yaml | 4 + .../distribution/distribution-templates.yaml | 2 + 4 files changed, 95 insertions(+), 88 deletions(-) diff --git a/jjb/integration/distribution/distribution-check-bootup.sh b/jjb/integration/distribution/distribution-check-bootup.sh index 2a333dea7..35076e1fc 100644 --- a/jjb/integration/distribution/distribution-check-bootup.sh +++ b/jjb/integration/distribution/distribution-check-bootup.sh @@ -73,42 +73,50 @@ ${WORKSPACE}/${BUNDLEFOLDER}/bin/start # No need for verbose printing during repeating operations. set +x -echo "Waiting for controller to come up..." -COUNT=0 -while true; do - RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)" - echo "${RESP}" - if [[ "${RESP}" == *"200"* ]]; then - echo Controller is UP - break - elif (( "${COUNT}" > 600 )); then - echo Timeout Controller DOWN - echo "Dumping Karaf log..." - cat "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Listing all open ports on controller system" - netstat -pnatu - exit 1 - else - COUNT=$(( ${COUNT} + 1 )) - sleep 1 - if [[ $(($COUNT % 5)) == 0 ]]; then - echo already waited ${COUNT} seconds... - fi - fi -done +function dump_log_and_exit { + echo "Dumping first 500K bytes of karaf log..." + head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" + echo "Dumping last 500K bytes of karaf log..." + tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" + cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" . + cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" . + exit 1 +} -echo "loading many features at once. Need to allow time for problems to show up in logs. cool down for 5 min ..." -COUNT="300" -while true; do - if (( "${COUNT}" <= "0" )); then - break - fi - echo "${COUNT} seconds yet to wait..." - sleep 10 - COUNT="$(( ${COUNT} - 10 ))" -done +echo "Waiting up to 5 minutes for controller to come up, checking every 5 seconds..." +if [ "${DISTROSTREAM}" == "carbon" ] || [ "${DISTROSTREAM}" == "nitrogen" ]; then + # Only oxygen and above have the infrautils.ready feature, so using REST API to determine if the controller is ready. + COUNT="0" + while true; do + COUNT=$(( ${COUNT} + 5 )) + sleep 5 + echo "already waited ${COUNT} seconds..." + RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)" + echo ${RESP} + if [[ ${RESP} == *"200"* ]]; then + echo "Controller is UP" + break + elif (( "${COUNT}" > "300" )); then + echo "Timeout Controller DOWN" + dump_log_and_exit + fi + done +else + COUNT="0" + while true; do + COUNT=$(( ${COUNT} + 5 )) + sleep 5 + echo "already waited ${COUNT} seconds..." + if grep --quiet 'org.opendaylight.infrautils.ready-impl.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then + echo "Controller is UP" + break + elif (( "${COUNT}" > "300" )); then + echo "Timeout Controller DOWN" + dump_log_and_exit + fi + done +fi -# End of repeating operations, enable verbose printing. set -x # echo "Checking OSGi bundles..." @@ -122,25 +130,13 @@ function exit_on_log_file_message { echo "looking for \"$1\" in karaf.log file" if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then echo ABORTING: found "$1" - echo "Dumping first 500K bytes of karaf log..." - head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Dumping last 500K bytes of karaf log..." - tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" . - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" . - exit 1 + dump_log_and_exit fi echo "looking for \"$1\" in karaf_console.log file" if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then echo ABORTING: found "$1" - echo "Dumping first 500K bytes of karaf log..." - head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" - echo "Dumping last 500K bytes of karaf log..." - tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" . - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" . - exit 1 + dump_log_and_exit fi } diff --git a/jjb/integration/distribution/distribution-deploy-verify.sh b/jjb/integration/distribution/distribution-deploy-verify.sh index c81ac6db6..ba28c7f64 100644 --- a/jjb/integration/distribution/distribution-deploy-verify.sh +++ b/jjb/integration/distribution/distribution-deploy-verify.sh @@ -61,34 +61,51 @@ mkdir -p ${WORKSPACE}/${BUNDLEFOLDER}/data/log echo "Starting controller..." ${WORKSPACE}/${BUNDLEFOLDER}/bin/start -echo "Waiting for controller to come up..." -COUNT=0 -while true; do - RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)" - echo "${RESP}" - if [[ "${RESP}" == *"200"* ]]; then - echo Controller is UP - break - elif (( "${COUNT}" > 600 )); then - echo Timeout Controller DOWN - echo "Dumping Karaf log..." - cat "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Listing all open ports on controller system" - netstat -pnatu - exit 1 - else - COUNT=$(( ${COUNT} + 1 )) - sleep 1 - if [[ $(($COUNT % 5)) == 0 ]]; then - echo already waited ${COUNT} seconds... - fi - fi -done +function dump_log_and_exit { + echo "Dumping first 500K bytes of karaf log..." + head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" + echo "Dumping last 500K bytes of karaf log..." + tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" + cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" . + cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" . + exit 1 +} -echo "loading many features at once. Need to allow time for problems to show up in logs. cool down for 5 min ..." -sleep 300 +echo "Waiting up to 5 minutes for controller to come up, checking every 5 seconds..." +if [ "${DISTROSTREAM}" == "carbon" ] || [ "${DISTROSTREAM}" == "nitrogen" ]; then + # Only oxygen and above have the infrautils.ready feature, so using REST API to determine if the controller is ready. + COUNT="0" + while true; do + COUNT=$(( ${COUNT} + 5 )) + sleep 5 + echo "already waited ${COUNT} seconds..." + RESP="$(curl --user admin:admin -sL -w "%{http_code} %{url_effective}\\n" http://localhost:8181/restconf/modules -o /dev/null || true)" + echo ${RESP} + if [[ ${RESP} == *"200"* ]]; then + echo "Controller is UP" + break + elif (( "${COUNT}" > "300" )); then + echo "Timeout Controller DOWN" + dump_log_and_exit + fi + done +else + COUNT="0" + while true; do + COUNT=$(( ${COUNT} + 5 )) + sleep 5 + echo "already waited ${COUNT} seconds..." + if grep --quiet 'org.opendaylight.infrautils.ready-impl.*System ready' "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then + echo "Controller is UP" + break + elif (( "${COUNT}" > "300" )); then + echo "Timeout Controller DOWN" + dump_log_and_exit + fi + done +fi -echo "Checking OSGi bundles..." +# echo "Checking OSGi bundles..." # sshpass seems to fail with new karaf version # sshpass -p karaf ${WORKSPACE}/${BUNDLEFOLDER}/bin/client -u karaf 'bundle:list' @@ -99,25 +116,13 @@ function exit_on_log_file_message { echo "looking for \"$1\" in karaf.log file" if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log"; then echo ABORTING: found "$1" - echo "Dumping first 500K bytes of karaf log..." - head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" - echo "Dumping last 500K bytes of karaf log..." - tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" . - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" . - exit 1 + dump_log_and_exit fi echo "looking for \"$1\" in karaf_console.log file" if grep --quiet "$1" "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log"; then echo ABORTING: found "$1" - echo "Dumping first 500K bytes of karaf log..." - head --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" - echo "Dumping last 500K bytes of karaf log..." - tail --bytes=500K "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log" . - cp "${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf_console.log" . - exit 1 + dump_log_and_exit fi } diff --git a/jjb/integration/distribution/distribution-jobs.yaml b/jjb/integration/distribution/distribution-jobs.yaml index 1ffd0ea7d..94e9f13cc 100644 --- a/jjb/integration/distribution/distribution-jobs.yaml +++ b/jjb/integration/distribution/distribution-jobs.yaml @@ -218,6 +218,8 @@ branch: '{branch}' refspec: 'refs/heads/{branch}' artifacts: '{archive-artifacts}' + - integration-distribution-stream: + stream: '{stream}' - distribution-karaf-version: karaf-version: '{karaf-version}' - maven-exec: @@ -296,6 +298,8 @@ branch: '{branch}' refspec: 'refs/heads/{branch}' artifacts: '{archive-artifacts}' + - integration-distribution-stream: + stream: '{stream}' - integration-distribution-branch: branch: '{branch}' - integration-bundle-url: diff --git a/jjb/integration/distribution/distribution-templates.yaml b/jjb/integration/distribution/distribution-templates.yaml index e7c0746e2..445361234 100644 --- a/jjb/integration/distribution/distribution-templates.yaml +++ b/jjb/integration/distribution/distribution-templates.yaml @@ -37,6 +37,8 @@ branch: '{branch}' refspec: 'refs/heads/{branch}' artifacts: '{archive-artifacts}' + - integration-distribution-stream: + stream: '{stream}' - integration-distribution-git-url: git-url: '{git-url}' - distribution-karaf-version: -- 2.36.6