From 44163ddb897d72c718187682a0ae3b0134d08875 Mon Sep 17 00:00:00 2001
From: Matt Welch <matt.welch@intel.com>
Date: Mon, 14 Aug 2017 16:59:58 -0400
Subject: [PATCH] enable automated setup of physical infrastructure

Ansible role to setup the physical infrastructure servers for the S3P
Scale testing framework.
These roles are refactored to improve structure for upstream.
Source is based on
https://github.com/intel-odl/socets/commit/975f5113e0ce16cd6ef1eeb03176230e4f6445c9
Consolidated bridge management and refactored for clarity & reuse.

Clean up infrastructure role
Improve execution idempotency by registering docker_updated_result and
only restarting docker daemon when needed.
Same with parse_apparmor_profile handler.
Added tags to most tasks for better separation of responsibilities.
Moved lab-specific variables tnd inventory to /etc/ansible/hosts.
Addes top-level site.yml to call infrastructure role.

fix docker proxy and 'restart docker' handler
Infra role will now setup proxies and only restart docker daemon on
change.
Refactor to move most variables into common role.
Updated handler

Removed trailing whitespace

Change-Id: I7956b5acba7d339af2e8abbb588c07f0e809de5c
Signed-off-by: Matt Welch <matt.welch@intel.com>
---
 roles/.gitignore                              |  1 +
 roles/common/vars/main.yml                    | 66 +++++++++++++++++++
 roles/examples/example_setup_infra.yml        |  5 ++
 roles/infra/files/docker-unconfined           | 20 ++++++
 roles/infra/handlers/main.yml                 | 10 +++
 .../infra/handlers/parse_apparmor_profile.yml |  9 +++
 roles/infra/tasks/configure_docker_daemon.yml | 13 ++++
 roles/infra/tasks/copy_apparmor_profile.yml   | 14 ++++
 roles/infra/tasks/create_bridges.yml          | 21 ++++++
 roles/infra/tasks/install_docker.yml          | 15 +++++
 roles/infra/tasks/install_packages.yml        | 26 ++++++++
 roles/infra/tasks/main.yml                    |  8 +++
 roles/infra/tasks/set_apt_proxy.yml           | 12 ++++
 roles/infra/tasks/set_bash_proxy.yml          |  9 +++
 roles/infra/tasks/set_docker_proxy.yml        | 23 +++++++
 roles/infra/tasks/set_proxies.yml             |  6 ++
 roles/infra/tasks/setup_bridge.yml            | 15 +++++
 roles/infra/tasks/setup_docker_daemon.yml     | 15 +++++
 roles/infra/tasks/update_etc_hosts.yml        | 15 +++++
 roles/infra/templates/daemon.json.j2          |  6 ++
 roles/infra/templates/http-proxy.conf.j2      |  3 +
 roles/infra/vars/main.yml                     | 15 +++++
 roles/infrastructure.yml                      |  5 ++
 roles/site.yml                                |  6 ++
 24 files changed, 338 insertions(+)
 create mode 100644 roles/.gitignore
 create mode 100644 roles/common/vars/main.yml
 create mode 100644 roles/examples/example_setup_infra.yml
 create mode 100644 roles/infra/files/docker-unconfined
 create mode 100644 roles/infra/handlers/main.yml
 create mode 100644 roles/infra/handlers/parse_apparmor_profile.yml
 create mode 100644 roles/infra/tasks/configure_docker_daemon.yml
 create mode 100644 roles/infra/tasks/copy_apparmor_profile.yml
 create mode 100644 roles/infra/tasks/create_bridges.yml
 create mode 100644 roles/infra/tasks/install_docker.yml
 create mode 100644 roles/infra/tasks/install_packages.yml
 create mode 100644 roles/infra/tasks/main.yml
 create mode 100644 roles/infra/tasks/set_apt_proxy.yml
 create mode 100644 roles/infra/tasks/set_bash_proxy.yml
 create mode 100644 roles/infra/tasks/set_docker_proxy.yml
 create mode 100644 roles/infra/tasks/set_proxies.yml
 create mode 100644 roles/infra/tasks/setup_bridge.yml
 create mode 100644 roles/infra/tasks/setup_docker_daemon.yml
 create mode 100644 roles/infra/tasks/update_etc_hosts.yml
 create mode 100644 roles/infra/templates/daemon.json.j2
 create mode 100644 roles/infra/templates/http-proxy.conf.j2
 create mode 100644 roles/infra/vars/main.yml
 create mode 100644 roles/infrastructure.yml
 create mode 100644 roles/site.yml

diff --git a/roles/.gitignore b/roles/.gitignore
new file mode 100644
index 0000000..a8b42eb
--- /dev/null
+++ b/roles/.gitignore
@@ -0,0 +1 @@
+*.retry
diff --git a/roles/common/vars/main.yml b/roles/common/vars/main.yml
new file mode 100644
index 0000000..00cd5c8
--- /dev/null
+++ b/roles/common/vars/main.yml
@@ -0,0 +1,66 @@
+# common variables
+---
+# network infrastructure
+## lab network resources
+# lab_http_proxy:
+# lab_https_proxy:
+# infrastructure_server:
+# registry_ip_address:
+# docker_registry_port:
+# docker_registry:
+
+
+## network configuration of host machines
+mgmt_iface: eno3
+data_iface: eno4
+management_bridge: br_mgmt
+data_bridge: br_data
+test_netmask: 16
+management_interface: "{{ mgmt_iface }}"
+data_interface: "{{ data_iface }}"
+management_subnet_prefix: "10.129"
+mgmt_ip_prefix: "10.129"
+management_subnet_netmask: "16"
+data_subnet_prefix: "10.130"
+data_ip_prefix: "10.130"
+data_subnet_netmask: "16"
+
+# IP address of the Linux bridge on the physical host
+mgmt_ip: "{{ mgmt_ip_prefix }}.{{ rackpos }}.1/{{ test_netmask }}"
+data_ip: "{{ data_ip_prefix }}.{{ rackpos }}.1/{{ test_netmask }}"
+
+# lab/default IP addresses of physical hosts
+mgmt_lab_ip_prefix: "10.11.26"
+data_lab_ip_prefix: "10.11.126"
+lab_netmask: 22
+# position and rackpos are an integer "index" of physical machines
+position: "{{ rackpos }}"
+# default IP addresses for lab machine interfaces
+mgmt_lab_ip: "{{ mgmt_lab_ip_prefix }}.{{ rackpos }}/{{ lab_netmask }}"
+data_lab_ip: "{{ data_lab_ip_prefix }}.{{ rackpos }}/{{ lab_netmask }}"
+
+# node operating system defaults
+# Variables for emulated framework containers
+## container configuration
+docker_systemd_version: "v0.1"
+compute_image: "s3p/compute"
+compute_version: "v0.5s"
+measure_image: "s3p/measure"
+measure_version: "v0.1.1"
+service_image: "s3p/service"
+service_version: "v0.4sc"
+control_node_image: "{{ docker_registry }}/{{ service_image }}:{{ service_version }}"
+
+## cluster configuration
+### the service_host ip should be conform to infrastructure routing
+#### service_host_phys_host is the host_index for the physical server
+service_host_phys_host: "20"
+service_host_container_index: "2"
+service_host_mgmt_ip: "{{ management_subnet_prefix }}.{{ service_host_phys_host }}.{{ service_host_container_index }}"
+control_node_container_name: "service-node"
+
+# network parameters for service and compute nodes
+lab_no_proxy: "localhost,10.0.0.0/8,192.168.0.0/16,172.17.0.0/16,127.0.0.1,127.0.0.0/8,{{ service_host_mgmt_ip }}"
+
+# OpenStack framework config
+use_odl_network: "True"
diff --git a/roles/examples/example_setup_infra.yml b/roles/examples/example_setup_infra.yml
new file mode 100644
index 0000000..d329e6e
--- /dev/null
+++ b/roles/examples/example_setup_infra.yml
@@ -0,0 +1,5 @@
+---
+- hosts: compute-node-hosts
+  become: yes
+  roles:
+    - role: infra
diff --git a/roles/infra/files/docker-unconfined b/roles/infra/files/docker-unconfined
new file mode 100644
index 0000000..1fd0800
--- /dev/null
+++ b/roles/infra/files/docker-unconfined
@@ -0,0 +1,20 @@
+# suggestion from bprodoehl
+# https://github.com/docker/docker/issues/5490
+# https://github.com/ewindisch/docker/commit/d45be4e65288f2978dfedd4a940f3debe128fd35
+include <tunables/global>
+
+profile docker-unconfined flags=(attach_disconnected,mediate_deleted) {
+  include <abstractions/base>
+  network,
+  capability,
+  file,
+  umount,
+  mount,
+  ptrace,
+  signal,
+  dbus,
+
+}
+
+# vim: ft=conf  :
+
diff --git a/roles/infra/handlers/main.yml b/roles/infra/handlers/main.yml
new file mode 100644
index 0000000..88b3726
--- /dev/null
+++ b/roles/infra/handlers/main.yml
@@ -0,0 +1,10 @@
+---
+- include: parse_apparmor_profile.yml
+
+- name: restart docker
+  tags: docker
+  systemd:
+    state: restarted
+    daemon_reload: yes
+    name: docker
+  listen: restart docker
diff --git a/roles/infra/handlers/parse_apparmor_profile.yml b/roles/infra/handlers/parse_apparmor_profile.yml
new file mode 100644
index 0000000..e5c74e3
--- /dev/null
+++ b/roles/infra/handlers/parse_apparmor_profile.yml
@@ -0,0 +1,9 @@
+- name: Parse apparmor profile
+  tags:
+    - files
+    - docker
+  shell: apparmor_parser -r -W "{{ profile_path }}"
+  listen: "Parse apparmor profile"
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/configure_docker_daemon.yml b/roles/infra/tasks/configure_docker_daemon.yml
new file mode 100644
index 0000000..9e1f6c8
--- /dev/null
+++ b/roles/infra/tasks/configure_docker_daemon.yml
@@ -0,0 +1,13 @@
+---
+- name: Configure docker daemon with daemon.json
+  tags:
+    - docker
+    - daemon
+    - configure
+  template:
+    src: templates/daemon.json.j2
+    dest: /etc/docker/daemon.json
+  register: docker_updated_result
+  notify:
+  - restart docker
+
diff --git a/roles/infra/tasks/copy_apparmor_profile.yml b/roles/infra/tasks/copy_apparmor_profile.yml
new file mode 100644
index 0000000..9939a27
--- /dev/null
+++ b/roles/infra/tasks/copy_apparmor_profile.yml
@@ -0,0 +1,14 @@
+- name: Copy docker-unconfined apparmor profile to host
+  tags:
+    - files
+    - security
+    - docker
+  copy:
+    src: files/docker-unconfined
+    dest: "{{ profile_path }}"
+    mode: 0755
+  notify:
+    - Parse apparmor profile
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/create_bridges.yml b/roles/infra/tasks/create_bridges.yml
new file mode 100644
index 0000000..e5b01f1
--- /dev/null
+++ b/roles/infra/tasks/create_bridges.yml
@@ -0,0 +1,21 @@
+---
+- name: Setup mgmt bridge
+  vars:
+    interface: "{{ management_interface }}"
+    bridge: "{{ management_bridge }}"
+    ip_addr_1: "{{ mgmt_lab_ip }}"
+    ip_addr_2: "{{ mgmt_ip }}"
+  include: setup_bridge.yml
+  when: "'{{ management_bridge }}' not in ansible_interfaces"
+
+- name: Setup data bridge
+  vars:
+    interface: "{{ data_interface }}"
+    bridge: "{{ data_bridge }}"
+    ip_addr_1: "{{ data_lab_ip }}"
+    ip_addr_2: "{{ data_ip }}"
+  include: setup_bridge.yml
+  when: "'{{ data_bridge }}' not in ansible_interfaces"
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/install_docker.yml b/roles/infra/tasks/install_docker.yml
new file mode 100644
index 0000000..83edfc9
--- /dev/null
+++ b/roles/infra/tasks/install_docker.yml
@@ -0,0 +1,15 @@
+---
+- name: Install the docker daemon
+  remote_user: root
+  become: yes
+  become_method: sudo
+  tags:
+    - docker
+    - packages
+    - installation
+  apt:
+    name: docker.io
+    state: present
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/install_packages.yml b/roles/infra/tasks/install_packages.yml
new file mode 100644
index 0000000..bb0e59b
--- /dev/null
+++ b/roles/infra/tasks/install_packages.yml
@@ -0,0 +1,26 @@
+---
+- name: Only run "update_cache=yes" if the last one is more than 3600 seconds ago
+  tags:
+    - packages
+    - update_cache
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+
+- name: Install packages if already installed ignore
+  tags:
+    - packages
+    - install
+  action: apt pkg={{item}} state=installed
+  with_items:
+       - git
+       - sshpass
+       - openssh-server
+       - tmux
+       - screen
+       - python
+       - vim
+       - python-docker
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/main.yml b/roles/infra/tasks/main.yml
new file mode 100644
index 0000000..240b437
--- /dev/null
+++ b/roles/infra/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- include: copy_apparmor_profile.yml
+- include: create_bridges.yml
+- include: install_packages.yml
+- include: setup_docker_daemon.yml
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/set_apt_proxy.yml b/roles/infra/tasks/set_apt_proxy.yml
new file mode 100644
index 0000000..bf81148
--- /dev/null
+++ b/roles/infra/tasks/set_apt_proxy.yml
@@ -0,0 +1,12 @@
+---
+- name: Set proxy in apt.conf
+  tags:
+    - proxy
+    - network
+    - packages
+  lineinfile:
+    dest: /etc/apt/apt.conf
+    regexp: '^Acquire::http::Proxy'
+    line: "Acquire::http::Proxy \"{{ lab_http_proxy }}\";"
+    create: yes
+    state: present
diff --git a/roles/infra/tasks/set_bash_proxy.yml b/roles/infra/tasks/set_bash_proxy.yml
new file mode 100644
index 0000000..df168bc
--- /dev/null
+++ b/roles/infra/tasks/set_bash_proxy.yml
@@ -0,0 +1,9 @@
+---
+- name: Set proxy in bash profile
+  tags:
+    - proxy
+    - network
+  lineinfile:
+    dest: /root/.profile
+    regexp: '^export http_proxy'
+    line: "export http_proxy={{ lab_http_proxy }}"
diff --git a/roles/infra/tasks/set_docker_proxy.yml b/roles/infra/tasks/set_docker_proxy.yml
new file mode 100644
index 0000000..0e3f0b5
--- /dev/null
+++ b/roles/infra/tasks/set_docker_proxy.yml
@@ -0,0 +1,23 @@
+---
+- name: Ensure systemd drop-in directory exists for docker
+  tags:
+    - docker
+    - daemon
+    - configure
+  file:
+    path: "{{ systemd_docker_dropin_dir }}"
+    state: directory
+    mode: 0755
+
+- name: Configure http_proxy & no_proxy for docker daemon
+  tags:
+    - docker
+    - daemon
+    - configure
+    - network
+  template:
+    src: ../templates/http-proxy.conf.j2
+    dest: "{{ systemd_docker_dropin_dir }}/http-proxy.conf"
+  register: docker_updated_result
+  notify: restart docker
+
diff --git a/roles/infra/tasks/set_proxies.yml b/roles/infra/tasks/set_proxies.yml
new file mode 100644
index 0000000..c68a0b2
--- /dev/null
+++ b/roles/infra/tasks/set_proxies.yml
@@ -0,0 +1,6 @@
+---
+  - include: set_bash_proxy.yml
+  - include: set_apt_proxy.yml
+  - include: set_docker_proxy.yml
+
+# vim: set et sw=2 ts=2 ai ft=yaml :
diff --git a/roles/infra/tasks/setup_bridge.yml b/roles/infra/tasks/setup_bridge.yml
new file mode 100644
index 0000000..0d526f8
--- /dev/null
+++ b/roles/infra/tasks/setup_bridge.yml
@@ -0,0 +1,15 @@
+---
+- name: "Setup Linux Bridge '{{ bridge }}' and bind it to interface '{{ interface }}'"
+  remote_user: root
+  become: yes
+  become_method: sudo
+  shell: |
+    ip addr flush "{{ interface }}"
+    ip addr flush "{{ bridge }}"
+    ip link set   "{{ interface }}" up
+    ip link set   "{{ interface }}" promisc on
+    brctl addbr   "{{ bridge }}"
+    brctl addif   "{{ bridge }}" "{{ interface }}"
+    ip addr add   "{{ ip_addr_1 }}" dev "{{ bridge }}"
+    ip addr add   "{{ ip_addr_2 }}" dev "{{ bridge }}"
+    ip link set   "{{ bridge }}" up
diff --git a/roles/infra/tasks/setup_docker_daemon.yml b/roles/infra/tasks/setup_docker_daemon.yml
new file mode 100644
index 0000000..e1b30b7
--- /dev/null
+++ b/roles/infra/tasks/setup_docker_daemon.yml
@@ -0,0 +1,15 @@
+---
+- include: set_proxies.yml
+  when: lab_http_proxy is defined
+
+- name: Install docker on host system
+  include: install_docker.yml
+
+- name: Update docker registry in /etc/hosts
+  include: update_etc_hosts.yml
+
+- name: Configure docker daemon
+  include: configure_docker_daemon.yml
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/tasks/update_etc_hosts.yml b/roles/infra/tasks/update_etc_hosts.yml
new file mode 100644
index 0000000..3488ebb
--- /dev/null
+++ b/roles/infra/tasks/update_etc_hosts.yml
@@ -0,0 +1,15 @@
+---
+- name: Add infra server (docker registry) to /etc/hosts
+  tags:
+    - network
+    - docker
+  lineinfile:
+    dest: /etc/hosts
+    regexp: "^.*{{ infrastructure_server }}"
+    line: "{{ registry_ip_address }}    {{ infrastructure_server }}"
+    state: present
+  register: docker_updated_result
+  notify: restart docker
+
+# vim: set et ts=2 sw=2 ai ft=yaml :
+
diff --git a/roles/infra/templates/daemon.json.j2 b/roles/infra/templates/daemon.json.j2
new file mode 100644
index 0000000..81990e5
--- /dev/null
+++ b/roles/infra/templates/daemon.json.j2
@@ -0,0 +1,6 @@
+{
+    "debug": {{ debug_docker }},
+    "insecure-registries": ["{{ docker_registry }}"],
+    "selinux-enabled": {{ enforce_os_security_enable }},
+    "storage-driver": "{{ docker_storage_driver }}"
+}
diff --git a/roles/infra/templates/http-proxy.conf.j2 b/roles/infra/templates/http-proxy.conf.j2
new file mode 100644
index 0000000..1786509
--- /dev/null
+++ b/roles/infra/templates/http-proxy.conf.j2
@@ -0,0 +1,3 @@
+[Service]
+Environment="HTTP_PROXY={{ lab_http_proxy }}" "NO_PROXY=localhost,127.0.0.1,10.0.0.0/8,{{ infrastructure_server }},{{ registry_ip_address }}"
+
diff --git a/roles/infra/vars/main.yml b/roles/infra/vars/main.yml
new file mode 100644
index 0000000..229e432
--- /dev/null
+++ b/roles/infra/vars/main.yml
@@ -0,0 +1,15 @@
+---
+
+# apparmor info
+profile_path: /root/workspace/docker-unconfined
+
+# docker configuration (daemon.json)
+systemd_docker_dropin_dir: '/etc/systemd/system/docker.service.d/'
+
+# docker registry in common/vars/main.yml
+debug_docker: 'true'
+docker_storage_driver: 'overlay2'
+enforce_os_security_enable: 'false'
+
+
+
diff --git a/roles/infrastructure.yml b/roles/infrastructure.yml
new file mode 100644
index 0000000..8fb5a39
--- /dev/null
+++ b/roles/infrastructure.yml
@@ -0,0 +1,5 @@
+---
+- hosts: cluster0
+  roles:
+    - common
+    - infra
diff --git a/roles/site.yml b/roles/site.yml
new file mode 100644
index 0000000..1d1251a
--- /dev/null
+++ b/roles/site.yml
@@ -0,0 +1,6 @@
+---
+# file: site.yml
+- include: infrastructure.yml
+
+# vim: set et ai sw=2 ts=2 :
+
-- 
2.36.6