X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=packer%2Fprovision%2Fbaseline.sh;h=aae30d44b1e3be34881adf56e5646a8ef1fedde7;hb=044cd3188c6361490d2cd8be3903aac7282026f8;hp=6daa4ef421c69253a78673cfddb1c7fdf5a1e12e;hpb=c5a37a5f85681c39cc0f58c07d5df376a8a32395;p=releng%2Fbuilder.git diff --git a/packer/provision/baseline.sh b/packer/provision/baseline.sh index 6daa4ef42..aae30d44b 100644 --- a/packer/provision/baseline.sh +++ b/packer/provision/baseline.sh @@ -5,6 +5,43 @@ # force any errors to cause the script and job to end in failure set -xeu -o pipefail +enable_service() { + # Enable services for Ubuntu instances + # We purposely want to allow globbing to build the package array + # shellcheck disable=SC2206 + services=($@) + + for service in "${services[@]}"; do + echo "---> Enable service: $service" + FACTER_OS=$(/usr/bin/facter operatingsystem | tr '[:upper:]' '[:lower:]') + FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease) + if [ "$FACTER_OS" == "centos" ]; then + systemctl enable "$service" + systemctl start "$service" + systemctl status "$service" + elif [ "$FACTER_OS" == "ubuntu" ]; then + case "$FACTER_OSVER" in + 14.04) + service "$service" start + service "$service" status + ;; + 16.04) + systemctl enable "$service" + systemctl start "$service" + systemctl status "$service" + ;; + *) + echo "---> Unknown Ubuntu version $FACTER_OSVER" + exit 1 + ;; + esac + else + echo "---> Unknown OS $FACTER_OS" + exit 1 + fi + done +} + ensure_kernel_install() { # Workaround for mkinitrd failing on occassion. # On CentOS 7 it seems like the kernel install can fail it's mkinitrd @@ -37,6 +74,8 @@ ensure_ubuntu_install() { # On Ubuntu sometimes the mirrors fail to install a package. This wrapper # checks that a package is successfully installed before moving on. + # We purposely want to allow globbing to build the package array + # shellcheck disable=SC2206 packages=($@) for pkg in "${packages[@]}" @@ -44,6 +83,11 @@ ensure_ubuntu_install() { # Retry installing package 5 times if necessary for i in {0..5} do + + # Wait for any background apt processes to finish before running apt + while pgrep apt > /dev/null; do sleep 1; done + + echo "$i: Installing $pkg" if [ "$(dpkg-query -W -f='${Status}' "$pkg" 2>/dev/null | grep -c "ok installed")" -eq 0 ]; then apt-cache policy "$pkg" apt-get install "$pkg" @@ -111,7 +155,23 @@ EOF # separate group installs from package installs since a non-existing # group with dnf based systems (F21+) will fail the install if such # a group does not exist - yum install -y unzip xz puppet git git-review perl-XML-XPath + yum install -y unzip xz git git-review perl-XML-XPath + + # facter is not installed by default on the base image and facter package + # gets removed when puppet4 is updated. To fix this use version of facter + # shipped with puppet4. + yum install -y facter + + # ensure facter is available in $PATH avoid failures in retry loop + export PATH="/opt/puppetlabs/bin/:$PATH" + + # Install puppet4 + rpm -ivh https://yum.puppetlabs.com/puppetlabs-release-pc1-el-7.noarch.rpm + yum -y install -y puppet-agent + + # Create symlink for facter and puppet + ln -sf /opt/puppetlabs/bin/facter /usr/bin/facter + ln -sf /opt/puppetlabs/puppet/bin/puppet /usr/bin/puppet # All of our systems require Java (because of Jenkins) # Install all versions of the OpenJDK devel but force 1.7.0 to be the @@ -120,10 +180,10 @@ EOF echo "---> Configuring OpenJDK" yum install -y 'java-*-openjdk-devel' - FACTER_OS=$(/usr/bin/facter operatingsystem) + FACTER_OS=$(/usr/bin/facter operatingsystem | tr '[:upper:]' '[:lower:]') FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease) case "$FACTER_OS" in - Fedora) + fedora) if [ "$FACTER_OSVER" -ge "21" ] then echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist" @@ -132,7 +192,7 @@ EOF alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 fi ;; - RedHat|CentOS) + redhat|centos) if [ "$(echo "$FACTER_OSVER" | cut -d'.' -f1)" -ge "7" ] then echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist" @@ -157,6 +217,10 @@ EOF # to initialize Heat template based systems. yum install -y jq + # Used py lftools to speend up some scripts + wget http://ftp.riken.jp/Linux/cern/centos/7/cern/x86_64/Packages/parallel-20150522-1.el7.cern.noarch.rpm + yum localinstall -y parallel-20150522-1.el7.cern.noarch.rpm + # Used by lftools scripts to parse XML yum install -y xmlstarlet @@ -171,12 +235,26 @@ EOF cabal install "Cabal<1.18" # Pull Cabal version that is capable of building shellcheck cabal install --bindir=/usr/local/bin "shellcheck-0.4.6" # Pin shellcheck version + # NLTK_DATA Cache: many jobs that use coala pull down nltk_data + wget -nv -O /opt/nltk_data.zip https://github.com/nltk/nltk_data/archive/gh-pages.zip + # --- END LFTOOLS DEPS ###################### # install haveged to avoid low entropy rejecting ssh connections yum install -y haveged systemctl enable haveged.service + + # Install sysstat + yum install -y sysstat + enable_service sysstat + + # Install python3 and dependencies, needed for Coala linting at least + yum install -y python34 + yum install -y python34-{devel,virtualenv,setuptools,pip} + + # Install python dependencies, useful generally + yum install -y python-{devel,virtualenv,setuptools,pip} } ubuntu_systems() { @@ -207,9 +285,6 @@ Dpkg::Options { EOF - # Add hostname to /etc/hosts to fix 'unable to resolve host' issue with sudo - sed -i "/127.0.0.1/s/$/ $(hostname)/" /etc/hosts - echo "---> Updating operating system" # add additional repositories @@ -220,26 +295,70 @@ EOF apt-get update -m apt-get upgrade -m apt-get dist-upgrade -m + apt-get install facter - ensure_ubuntu_install unzip xz-utils puppet git libxml-xpath-perl + # facter is not installed by default on the base image and facter package + # gets removed when puppet4 is updated. To fix this use version of facter + # shipped with puppet4. + # ensure facter is available in $PATH avoid failures in retry loop + export PATH="/opt/puppetlabs/bin/:$PATH" - # install Java 7 - echo "---> Configuring OpenJDK" - FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease) + ensure_ubuntu_install unzip xz-utils git libxml-xpath-perl + + # Install python3 and dependencies, needed for Coala linting + ensure_ubuntu_install python3 + ensure_ubuntu_install python3-{dev,setuptools,pip} + + # Install python and dependencies + ensure_ubuntu_install python-{dev,virtualenv,setuptools,pip} + + FACTER_OSVER=$(facter operatingsystemrelease) case "$FACTER_OSVER" in 14.04) + echo "---> Installing OpenJDK" apt-get install openjdk-7-jdk # make jdk8 available add-apt-repository -y ppa:openjdk-r/ppa apt-get update # We need to force openjdk-8-jdk to install apt-get install openjdk-8-jdk + echo "---> Configuring OpenJDK" # make sure that we still default to openjdk 7 update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java update-alternatives --set javac /usr/lib/jvm/java-7-openjdk-amd64/bin/javac + + echo "---> Install puppet repository for 14.04" + apt-get purge puppet puppet-common + wget https://apt.puppetlabs.com/puppetlabs-release-pc1-trusty.deb + dpkg -i puppetlabs-release-pc1-trusty.deb + + echo "---> Installing puppet" + apt-get update -m + ensure_ubuntu_install puppet-agent + # Create symlink for facter and puppet + ln -sf /opt/puppetlabs/bin/facter /usr/bin/facter + ln -sf /opt/puppetlabs/puppet/bin/puppet /usr/bin/puppet ;; 16.04) + echo "---> Installing OpenJDK" apt-get install openjdk-8-jdk + + echo "---> Install puppet4 repository for 16.04" + apt-get purge puppet puppet-common + wget https://apt.puppetlabs.com/puppetlabs-release-pc1-xenial.deb + dpkg -i puppetlabs-release-pc1-xenial.deb + + echo "---> Installing puppet" + apt-get update -m + ensure_ubuntu_install puppet-agent + + # Create symlink for facter and puppet + ln -sf /opt/puppetlabs/bin/facter /usr/bin/facter + ln -sf /opt/puppetlabs/puppet/bin/puppet /usr/bin/puppet + + echo "---> Installing python3 virtualenv" + # python3-virtualenv is available starting with 16.04. + ensure_ubuntu_install python3-virtualenv ;; *) echo "---> Unknown Ubuntu version $FACTER_OSVER" @@ -257,6 +376,9 @@ EOF # to initialize Heat template based systems. ensure_ubuntu_install jq + # Used py lftools to speend up some scripts + ensure_ubuntu_install parallel + # Used by lftools scripts to parse XML ensure_ubuntu_install xmlstarlet @@ -270,17 +392,29 @@ EOF cabal update cabal install --bindir=/usr/local/bin "shellcheck-0.4.6" # Pin shellcheck version + # NLTK_DATA Cache: many jobs that use coala pull down nltk_data + wget -nv -O /opt/nltk_data.zip https://github.com/nltk/nltk_data/archive/gh-pages.zip + # --- END LFTOOLS DEPS ###################### - # install haveged to avoid low entropy rejecting ssh connections + # Install sysstat + ensure_ubuntu_install sysstat + sed -i 's/ENABLED="false"/ENABLED="true"/' /etc/default/sysstat + enable_service sysstat + + # Install haveged to avoid low entropy rejecting ssh connections apt-get install haveged update-rc.d haveged defaults - # disable unattended upgrades & daily updates + # Disable unattended upgrades & daily updates echo '---> Disabling automatic daily upgrades' sed -ine 's/"1"/"0"/g' /etc/apt/apt.conf.d/10periodic echo 'APT::Periodic::Unattended-Upgrade "0";' >> /etc/apt/apt.conf.d/10periodic + + # Install packaging job dependencies for building debs + ensure_ubuntu_install build-essential devscripts equivs dh-systemd python-yaml \ + python-jinja2 gdebi } all_systems() { @@ -292,9 +426,9 @@ all_systems() { # Do any Distro specific installations here echo "Checking distribution" - FACTER_OS=$(/usr/bin/facter operatingsystem) + FACTER_OS=$(/usr/bin/facter operatingsystem | tr '[:upper:]' '[:lower:]') case "$FACTER_OS" in - RedHat|CentOS) + redhat|centos) if [ "$(/usr/bin/facter operatingsystemrelease | /bin/cut -d '.' -f1)" = "7" ]; then echo echo "---> CentOS 7" @@ -311,6 +445,11 @@ all_systems() { echo "No extra steps for $FACTER_OS" ;; esac + + # Update /etc/nss-switch.conf to map hostname with IP instead of using `localhost` + # from /etc/hosts which is required by some of the Java API's to avoid + # Java UnknownHostException: "Name or service not known" error. + sed -i "/^hosts:/s/$/ myhostname/" /etc/nsswitch.conf } echo "---> Attempting to detect OS"