* CentOS 7 - devstack - 20170117-0003
* CentOS 7 - devstack - 20170120-1710
* CentOS 7 - devstack - 20170210-1356
+* CentOS 7 - devstack - 20170221-1719
* CentOS 7 - devstack - newton - 20170117-0005
* CentOS 7 - devstack-mitaka - 20170130-0523
* CentOS 7 - devstack-newton - 20170130-0426
- shell: |
./scripts/fix-relativepaths.sh
+- builder:
+ # include-raw-autorelease-notify-build-failure.sh searches console log for
+ # failures and emails the status to the release mailing list.
+ name: opendaylight-infra-notify-status
+ builders:
+ - shell:
+ !include-raw-escape: include-raw-autorelease-notify-build-failure.sh
global-settings: 'odl-global-settings'
- autorelease-maven-sources-post-process
- autorelease-sys-stats
+ - opendaylight-infra-notify-status
- shell: |
mkdir -p archives/
cp *.log *.prop $_
--- /dev/null
+#!/bin/bash -x
+# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+#RELEASE_EMAIL="release@lists.opendaylight.org"
+RELEASE_EMAIL="abelur@linuxfoundation.org, thanh.ha@linuxfoundation.org"
+ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER"
+CONSOLE_LOG="/tmp/autorelease-build.log"
+
+BODY="Please refer to the logs server URL for console logs when possible
+and use the Jenkins Build URL as a last resort.
+
+Console Logs URL:
+https://logs.opendaylight.org/$SILO/$ARCHIVES_DIR
+
+Jenkins Build URL:
+$BUILD_URL"
+
+# get console logs
+wget -O $CONSOLE_LOG ${BUILD_URL}consoleText
+
+# get the failed project or artifactid
+TEMP=`awk '/Reactor Summary:/{flag=1;next} \
+ /Final Memory:/{flag=0}flag' $CONSOLE_LOGS \
+ | grep '. FAILURE \[' | awk -F'[].]' '{gsub(/ /, "", $2); print $2 }'`
+
+# check for project format
+if [[ ${TEMP} =~ .*::*.*::*. ]]; then
+ # extract project and artifactid from full format
+ PROJECT=`echo ${TEMP} | awk -F'::' '{ print $2 }'`
+ ARTIFACTID=`echo ${TEMP} |awk -F'::' '{ print $3 }'`
+else
+ # set ARTIFACTID to partial format
+ ARTIFACTID=${TEMP}
+fi
+
+# check if remote staging is complete successfully
+BUILD_STATUS=`awk '/\[INFO\] Remote staging finished/{flag=1;next} \
+ /Total time:/{flag=0}flag' $CONSOLE_LOG \
+ | grep '\] BUILD' | awk '{print $3}'`
+
+if [ ! -z "${ARTIFACTID}" ] && [[ "${BUILD_STATUS}" != "SUCCESS" ]]; then
+ # project search pattern should handle both scenarios
+ # 1. Full format: ODL :: $PROJECT :: $ARTIFACTID
+ # 2. Partial format: Building $ARTIFACTID
+ awk "/\[INFO\] Building ${ARTIFACTID} / || /ODL :: ${PROJECT} :: ${ARTIFACTID} /{flag=1;next} \
+ /Reactor Summary:/{flag=0}flag" $CONSOLE_LOG > /tmp/error_msg
+
+ if [ -z "${PROJECT}" ]; then
+ PROJECT=${ARTIFACTID}
+ # TODO: unset the below line when ready to deploy to real lists
+ # RELEASE_EMAIL = "${RELEASE_EMAIL}, ${PROJECT}-dev@opendaylight.org"
+ fi
+
+ SUBJECT="[release] Autorelease build failure: ${PROJECT}"
+
+ echo "${BODY}" | mail -A /tmp/error_msg -s "${SUBJECT}" "${RELEASE_EMAIL}"
+fi
+
+rm $CONSOLE_LOG
echo "Extracting the last distribution found on nexus..."
unzip -q $BUNDLE
mv $BUNDLEFOLDER /tmp/distro_old
+rm $BUNDLE
echo "Extracting the distribution just created by this job..."
NEW_DISTRO=$(find $WORKSPACE -name distribution-karaf*.zip)
cd test/tools/distchanges
mkdir -p $WORKSPACE/archives
-python distcompare.py -r ssh://jenkins-$SILO@git.opendaylight.org:29418 | tee $WORKSPACE/archives/dist_diff.txt
-# TODO: the output of the above command is not *friendly* for the reader because the most important info
-# is listed last. This is fine/best for command line output, but for keeping in a file it would be better
-# to put the summary at the beginning of the file. Some bash magic can be done here to make that happen.
+# Full output of compare tool will be in temp file /tmp/dist_diff.txt
+# The file/report to be archived will only list the distribution in the comparison and the patches that
+# are different.
+python distcompare.py -r ssh://jenkins-$SILO@git.opendaylight.org:29418 | tee /tmp/dist_diff.txt
+echo -e "Patch differences listed are in comparison to:\n\t$ACTUALBUNDLEURL\n\n" > $WORKSPACE/archives/distribution_differences.txt
+sed -ne '/Patch differences/,$ p' /tmp/dist_diff.txt >> $WORKSPACE/archives/distribution_differences.txt
sed -ie 's/log4j.appender.out.maxBackupIndex=10/log4j.appender.out.maxBackupIndex=1/g' \${LOGCONF}
# FIXME: Make log size limit configurable from build parameter.
sed -ie 's/log4j.appender.out.maxFileSize=1MB/log4j.appender.out.maxFileSize=30GB/g' \${LOGCONF}
+echo "log4j.logger.org.opendaylight.yangtools.yang.parser.repo.YangTextSchemaContextResolver = WARN" >> \${LOGCONF}
cat \${LOGCONF}
echo "Configure java home and max memory..."
create_control_node_local_conf
scp ${WORKSPACE}/local.conf_control ${OPENSTACK_CONTROL_NODE_IP}:/opt/stack/devstack/local.conf
+
+# Workworund for successful stacking with Mitaka
if [ "${ODL_ML2_BRANCH}" == "stable/mitaka" ]; then
-ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt"
-ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://github.com/openstack/python-openstacksdk; cd python-openstacksdk; sudo python setup.py install"
+
+ # Workaround for problems with latest versions/specified versions in requirements of openstack
+ # Openstacksdk,libvirt-python -> the current version does not work with Mitaka diue to some requirements
+ # conflict and breaks when trying to stack
+ # paramiko -> Problems with tempest tests due to paramiko incompatibility with pycrypto.
+ # the problem has been solved with version 1.17. If the latest version of paramiko is used, it causes
+ # other timeout problems
+ ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://git.openstack.org/openstack/requirements; cd requirements; git checkout stable/mitaka; sed -i /openstacksdk/d upper-constraints.txt; sed -i /libvirt-python/d upper-constraints.txt; sed -i /paramiko/d upper-constraints.txt"
+ ssh ${OPENSTACK_CONTROL_NODE_IP} "sudo pip install deprecation"
+ ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://github.com/openstack/python-openstacksdk; cd python-openstacksdk; sudo python setup.py install"
+ ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack; git clone https://github.com/paramiko/paramiko; cd paramiko; git checkout 1.17; sudo python setup.py install"
fi
ssh ${OPENSTACK_CONTROL_NODE_IP} "cd /opt/stack/devstack; nohup ./stack.sh > /opt/stack/devstack/nohup.out 2>&1 &"
project: 'netvirt'
project-name: 'netvirt'
branch: 'topic/vpp'
+ distribution_branch: 'master'
jdk: openjdk8
jdks:
- openjdk8
JOB_SUM=`echo $JOB_NAME | sum | awk '{{ print $1 }}'`
VM_NAME="$JOB_SUM-$BUILD_NUMBER"
+OS_TIMEOUT=10 # Minutes to wait for OpenStack VM to come online
+STACK_RETRIES=3 # Number of times to retry creating a stack before fully giving up
+STACK_SUCCESSFUL=false
# seq X refers to waiting for X minutes for OpenStack to return
# a status that is not CREATE_IN_PROGRESS before giving up.
-OS_TIMEOUT=15 # Minutes to wait for OpenStack VM to come online
openstack --os-cloud rackspace limits show --absolute
openstack --os-cloud rackspace limits show --rate
-openstack --os-cloud rackspace stack create --timeout $OS_TIMEOUT -t {stack-template} -e $WORKSPACE/opendaylight-infra-environment.yaml --parameter "job_name=$VM_NAME" --parameter "silo=$SILO" $STACK_NAME
-echo "Waiting for $OS_TIMEOUT minutes to create $STACK_NAME."
-for i in `seq $OS_TIMEOUT`; do
- sleep 60
- OS_STATUS=`openstack --os-cloud rackspace stack show -f json -c stack_status $STACK_NAME | jq -r '.stack_status'`
+echo "Trying up to $STACK_RETRIES times to create $STACK_NAME."
+for try in `seq $STACK_RETRIES`; do
+ openstack --os-cloud rackspace stack create --timeout $OS_TIMEOUT -t csit-2-instance-type.yaml -e $WORKSPACE/opendaylight-infra-environment.yaml --parameter "job_name=$VM_NAME" --parameter "silo=$SILO" $STACK_NAME
+ openstack --os-cloud rackspace stack list
+ echo "Waiting for $OS_TIMEOUT minutes to create $STACK_NAME."
+ for i in `seq $OS_TIMEOUT`; do
+ sleep 60
+ OS_STATUS=`openstack --os-cloud rackspace stack show -f json -c stack_status $STACK_NAME | jq -r '.stack_status'`
- case "$OS_STATUS" in
- CREATE_COMPLETE)
- echo "Stack initialized on infrastructure successful."
- break
- ;;
- CREATE_FAILED)
- echo "ERROR: Failed to initialize infrastructure. Quitting..."
- exit 1
- ;;
- CREATE_IN_PROGRESS)
- echo "Waiting to initialize infrastructure."
- continue
- ;;
- *)
- echo "Unexpected status: $OS_STATUS"
- exit 1
- ;;
- esac
+ case "$OS_STATUS" in
+ CREATE_COMPLETE)
+ echo "Stack initialized on infrastructure successful."
+ STACK_SUCCESSFUL=true
+ break
+ ;;
+ CREATE_FAILED)
+ echo "ERROR: Failed to initialize infrastructure. Deleting stack and possibly retrying to create..."
+ openstack --os-cloud rackspace stack list
+ openstack --os-cloud rackspace stack delete --yes $STACK_NAME
+ openstack --os-cloud rackspace stack show $STACK_NAME
+ # after stack delete, poll for 10m to know when stack is fully removed
+ # the logic here is that when "stack show $STACK_NAME" does not contain $STACK_NAME
+ # we assume it's successfully deleted and we can break to retry
+ for i in `seq 20`; do
+ sleep 30;
+ STACK_SHOW=$(openstack --os-cloud rackspace stack show $STACK_NAME)
+ echo $STACK_SHOW
+ if [[ $STACK_SHOW == *"DELETE_FAILED"* ]]; then
+ echo "stack delete failed. trying to stack abandon now"
+ openstack --os-cloud rackspace stack abandon $STACK_NAME
+ STACK_SHOW=$(openstack --os-cloud rackspace stack show $STACK_NAME)
+ echo $STACK_SHOW
+ fi
+ if [[ $STACK_SHOW != *"$STACK_NAME"* ]]; then
+ echo "stack show on $STACK_NAME came back empty. Assuming successful delete"
+ break
+ fi
+ done
+ # if we still see $STACK_NAME in $STACK_SHOW it means the delete hasn't fully
+ # worked and we can exit forcefully
+ if [[ $STACK_SHOW == *"$STACK_NAME"* ]]; then
+ echo "stack $STACK_NAME still in stack show output after polling. Quitting!"
+ exit 1
+ fi
+ break
+ ;;
+ CREATE_IN_PROGRESS)
+ echo "Waiting to initialize infrastructure."
+ continue
+ ;;
+ *)
+ echo "Unexpected status: $OS_STATUS"
+ exit 1
+ ;;
+ esac
+ done
+ if $STACK_SUCCESSFUL; then
+ break
+ fi
done
# capture stack info in console logs
openstack --os-cloud rackspace stack show $STACK_NAME
+
+if ! $STACK_SUCCESSFUL; then
+ exit 1
+fi
# openstack-infra-parameters defaults
archive-artifacts: ''
branch: master
+ distribution_branch: '$GERRIT_BRANCH'
# Distribution bundle URL
bundleurl: 'last'
odl_system_image: CentOS 7 - java-builder - 20170126-0058
openstack_system_count: 1
openstack_system_flavor: 8 GB General Purpose v1
- openstack_system_image: CentOS 7 - devstack - 20170210-1356
+ openstack_system_image: CentOS 7 - devstack - 20170221-1719
tools_system_count: 1
tools_system_flavor: 2 GB General Purpose v1
tools_system_image: Ubuntu 14.04 - mininet - 20170210-0439
description: 'Parameter to identify an ODL Gerrit project'
- string:
name: ARCHIVE_ARTIFACTS
- default: '{artifacts} **/target/surefire-reports/*-output.txt **/hs_err_*.log'
+ default: '{artifacts} **/target/surefire-reports/*-output.txt **/hs_err_*.log **/target/feature/feature.xml'
description: 'Space separated glob patterns for artifacts to archive into logs.opendaylight.org'
- string:
name: GERRIT_PROJECT
refspec: '$GERRIT_REFSPEC'
branch: '{branch}'
- integration-distribution-scm:
- branch: '{branch}'
+ branch: '{distribution_branch}'
wrappers:
- opendaylight-infra-wrappers:
openstack-branch: 'stable/mitaka'
odl-ml2-branch: 'stable/mitaka'
odl-ml2-driver-version: 'v1'
+ openstack_system_image: 'CentOS 7 - devstack-mitaka - 20170210-1356'
- newton:
openstack-branch: 'stable/newton'
- odl-ml2-branch: 'master'
+ odl-ml2-branch: 'stable/newton'
odl-ml2-driver-version: 'v1'
+ openstack_system_image: 'CentOS 7 - devstack-newton - 20170210-1344'
schedule: ''
key_name: { get_param: ssh_key }
user_data: |
#!/bin/bash
+ until ping -c1 git.opendaylight.org &>/dev/null; do echo "Waiting until git.opendaylight.org is resolvable..."; done
git clone https://git.opendaylight.org/gerrit/releng/builder /builder
/builder/jenkins-scripts/jenkins-init-script.sh
echo "---> Updating operating system"
yum clean all
yum install -y deltarpm
+
+ # Workaround for kernel panic issue that appears sometimes after kernel update
+ # https://www.centos.org/forums/viewtopic.php?t=22425
+ yum remove -y kernel
yum update -y
+ yum install -y kernel
# add in components we need or want on systems
echo "---> Installing base packages"
Code Review / releng / builder.git / commitdiff