Images with pre-loaded devstack pips and yums

This patch adds templates and a provisioning
script to build images that are pre-installed
with (a lot of) the various pip and yum pkgs
required for running devstack.

This aims to (hopefully) reduce the run time of CSIT.

Note that this is a just a first step. The low hanging
fruit. There are more packages that are installed by
openstack projects that can be preinstalled here as well.

Change-Id: Ic837dd2ef7a96141130bb0d71f578cd06de5f1f4
Signed-off-by: Josh <jhershbe@redhat.com>

diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml
index bbd6a00c8a303f8598f7be5412c6718fca13ce9b..5c0b0b0ecd3e5bd408fa41dbfeaa34be7e7939e7 100644 (file)
--- a/jjb/releng-jobs.yaml
+++ b/jjb/releng-jobs.yaml
@@ -15,6 +15,8 @@
 
     templates:
         - devstack
+        - devstack-pre-pip-mitaka
+        - devstack-pre-pip-newton
         - docker
         - gbp
         - java-builder
@@ -34,10 +36,12 @@
           templates: mininet-ovs-2.5
         - platforms: ubuntu-14.04
           templates: java-builder
-        - platforms: centos
-          templates: gbp
         - platforms: ubuntu-14.04
           templates: robot
+        - platforms: ubuntu-14.04
+          templates: devstack-pre-pip-mitaka
+        - platforms: ubuntu-14.04
+          templates: devstack-pre-pip-newton
 
     project-type: freestyle
     archive-artifacts: '**/*.log'

diff --git a/packer/provision/devstack-pre-pip.sh b/packer/provision/devstack-pre-pip.sh
new file mode 100644 (file)
index 0000000..936ac3b
--- /dev/null
+++ b/packer/provision/devstack-pre-pip.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+echo '---> Installing non-baseline requirements'
+yum install -y deltarpm python{,-{crypto,devel,lxml,setuptools}} \
+    @development {lib{xml2,xslt,ffi},openssl}-devel git wget
+
+echo '---> Updating net link setup'
+if [ ! -f /etc/udev/rules.d/80-net-setup-link.rules ]; then
+    ln -s /dev/null /etc/udev/rules.d/80-net-setup-link.rules
+fi
+
+echo '---> Pre-installing yum and pip packages'
+projs="requirements keystone glance cinder neutron nova horizon"
+branch=${os_branch}
+
+wget https://bootstrap.pypa.io/get-pip.py
+python get-pip.py
+
+mkdir tmp
+cd tmp
+
+git clone https://github.com/openstack-dev/devstack.git
+(cd devstack && git checkout ${branch})
+sed -e 's/#.*//' devstack/files/rpms/general | xargs yum install -y
+
+base_url=https://github.com/openstack/
+for proj in $projs
+do
+    git clone ${base_url}${proj}
+    (cd ${proj} && git checkout ${branch})
+    pip install -c requirements/upper-constraints.txt -e ${proj}
+    pip install -c requirements/upper-constraints.txt -r ${proj}/test-requirements.txt
+done
+
+cd $OLDPWD
+rm -fr tmp
+
+# vim: sw=4 ts=4 sts=4 et :

diff --git a/packer/templates/devstack-pre-pip-mitaka.json b/packer/templates/devstack-pre-pip-mitaka.json
new file mode 100644 (file)
index 0000000..a6de66f
--- /dev/null
+++ b/packer/templates/devstack-pre-pip-mitaka.json
@@ -0,0 +1,71 @@
+{
+  "variables": {
+    "public_base_image": null,
+    "public_network": null,
+    "public_cloud_user": null,
+    "public_auth_url": null,
+    "public_tenant": null,
+    "public_user": null,
+    "public_pass": null,
+
+    "private_base_image": null,
+    "private_network": null,
+    "private_cloud_user": null,
+    "private_auth_url": null,
+    "private_tenant": null,
+    "private_user": null,
+    "private_pass": null,
+
+    "distro": null,
+    "cloud_user_data": null
+  },
+  "builders": [
+    {
+      "type": "openstack",
+      "name": "public_cloud",
+      "identity_endpoint": "{{user `public_auth_url`}}",
+      "tenant_name": "{{user `public_tenant`}}",
+      "username": "{{user `public_user`}}",
+      "password": "{{user `public_pass`}}",
+      "region": "DFW",
+      "ssh_username": "{{user `public_cloud_user`}}",
+      "image_name": "{{user `distro`}} - devstack - mitaka - {{isotime \"20060102-1504\"}}",
+      "source_image_name": "{{user `public_base_image`}}",
+      "flavor": "general1-1",
+      "networks": [
+        "{{user `public_network`}}"
+      ],
+      "user_data_file": "{{user `cloud_user_data`}}"
+    },
+    {
+      "type": "openstack",
+      "name": "private_cloud",
+      "identity_endpoint": "{{user `private_auth_url`}}",
+      "tenant_name": "{{user `private_tenant`}}",
+      "username": "{{user `private_user`}}",
+      "password": "{{user `private_pass`}}",
+      "ssh_username": "{{user `private_cloud_user`}}",
+      "image_name": "{{user `distro`}} - devstack - mitaka - {{isotime \"20060102-1504\"}}",
+      "source_image_name": "{{user `private_base_image`}}",
+      "flavor": "general1-1",
+      "networks": [
+        "{{user `private_network`}}"
+      ],
+      "user_data_file": "{{user `cloud_user_data`}}"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "environment_vars": [
+        "os_branch=stable/mitaka"
+      ],
+      "scripts": [
+        "provision/baseline.sh",
+        "provision/devstack-pre-pip.sh",
+        "provision/system_reseal.sh"
+      ],
+      "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
+    }
+  ]
+}

diff --git a/packer/templates/devstack-pre-pip-newton.json b/packer/templates/devstack-pre-pip-newton.json
new file mode 100644 (file)
index 0000000..a97d826
--- /dev/null
+++ b/packer/templates/devstack-pre-pip-newton.json
@@ -0,0 +1,71 @@
+{
+  "variables": {
+    "public_base_image": null,
+    "public_network": null,
+    "public_cloud_user": null,
+    "public_auth_url": null,
+    "public_tenant": null,
+    "public_user": null,
+    "public_pass": null,
+
+    "private_base_image": null,
+    "private_network": null,
+    "private_cloud_user": null,
+    "private_auth_url": null,
+    "private_tenant": null,
+    "private_user": null,
+    "private_pass": null,
+
+    "distro": null,
+    "cloud_user_data": null
+  },
+  "builders": [
+    {
+      "type": "openstack",
+      "name": "public_cloud",
+      "identity_endpoint": "{{user `public_auth_url`}}",
+      "tenant_name": "{{user `public_tenant`}}",
+      "username": "{{user `public_user`}}",
+      "password": "{{user `public_pass`}}",
+      "region": "DFW",
+      "ssh_username": "{{user `public_cloud_user`}}",
+      "image_name": "{{user `distro`}} - devstack - newton - {{isotime \"20060102-1504\"}}",
+      "source_image_name": "{{user `public_base_image`}}",
+      "flavor": "general1-1",
+      "networks": [
+        "{{user `public_network`}}"
+      ],
+      "user_data_file": "{{user `cloud_user_data`}}"
+    },
+    {
+      "type": "openstack",
+      "name": "private_cloud",
+      "identity_endpoint": "{{user `private_auth_url`}}",
+      "tenant_name": "{{user `private_tenant`}}",
+      "username": "{{user `private_user`}}",
+      "password": "{{user `private_pass`}}",
+      "ssh_username": "{{user `private_cloud_user`}}",
+      "image_name": "{{user `distro`}} - devstack - newton - {{isotime \"20060102-1504\"}}",
+      "source_image_name": "{{user `private_base_image`}}",
+      "flavor": "general1-1",
+      "networks": [
+        "{{user `private_network`}}"
+      ],
+      "user_data_file": "{{user `cloud_user_data`}}"
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "shell",
+      "environment_vars": [
+        "os_branch=stable/newton"
+      ],
+      "scripts": [
+        "provision/baseline.sh",
+        "provision/devstack-pre-pip.sh",
+        "provision/system_reseal.sh"
+      ],
+      "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi"
+    }
+  ]
+}