tools/clustering/cluster-deployer/deploy.py

   1 #!/usr/bin/env python
   2 #
   3 # This script deploys a cluster
   4 # -------------------------------------------
   5 #
   6 # Pre-requisites
   7 # - Python 2.7
   8 # - SSHLibrary (pip install robotframework-sshlibrary)
   9 # - pystache (pip install pystache)
  10 # - argparse (pip install argparse)
  11 #
  12 # The input that this script will take is as follows,
  13 #
  14 # - A comma separated list of ip addresses/hostnames for each host on which
  15 #   the distribution needs to be deployed
  16 # - The replication factor to be used
  17 # - The ssh username/password of the remote host(s). Note that this should be
  18 #   the same for each host
  19 # - The name of the template to be used.
  20 #   Note that this template name should match the name of a template folder in
  21 #   the templates directory.
  22 #   The templates directory can be found in the same directory as this script.
  23 #
  24 # Here are the things it will do,
  25 #
  26 # - Copy over a distribution of opendaylight to the remote host
  27 # - Create a timestamped directory on the remote host
  28 # - Unzip the distribution to the timestamped directory
  29 # - Copy over the template substituted configuration files to the appropriate
  30 #   location on the remote host
  31 # - Create a symlink to the timestamped directory
  32 # - Start karaf
  33 #
  34 # ----------------------------------------------------------------------------
  35
  36 import argparse
  37 import time
  38 import pystache
  39 import os
  40 import sys
  41 import random
  42 import re
  43 from remote_host import RemoteHost
  44
  45 parser = argparse.ArgumentParser(description='Cluster Deployer')
  46 parser.add_argument("--distribution", default="",
  47                     help="the absolute path of the distribution on the local "
  48                          "host that needs to be deployed. (Must contain "
  49                          "version in the form: \"<#>.<#>.<#>-<name>\", e.g. "
  50                          "0.2.0-SNAPSHOT)",
  51                     required=True)
  52 parser.add_argument("--rootdir", default="/root",
  53                     help="the root directory on the remote host where the "
  54                          "distribution is to be deployed",
  55                     required=True)
  56 parser.add_argument("--hosts", default="", help="a comma separated list of "
  57                                                 "host names or ip addresses",
  58                     required=True)
  59 parser.add_argument("--clean", action="store_true", default=False,
  60                     help="clean the deployment on the remote host")
  61 parser.add_argument("--template", default="openflow",
  62                     help="the name of the template to be used. "
  63                     "This name should match a folder in the templates "
  64                          "directory.")
  65 parser.add_argument("--rf", default=3, type=int,
  66                     help="replication factor. This is the number of replicas "
  67                          "that should be created for each shard.")
  68 parser.add_argument("--user", default="root", help="the SSH username for the "
  69                                                    "remote host(s)")
  70 parser.add_argument("--password", default="Ecp123",
  71                     help="the SSH password for the remote host(s)")
  72 args = parser.parse_args()
  73
  74
  75 #
  76 # The TemplateRenderer provides methods to render a template
  77 #
  78 class TemplateRenderer:
  79     def __init__(self, template):
  80         self.cwd = os.getcwd()
  81         self.template_root = self.cwd + "/templates/" + template + "/"
  82
  83     def render(self, template_path, output_path, variables=None):
  84         if variables is None:
  85             variables = {}
  86
  87         if os.path.exists(self.template_root + template_path) is False:
  88             return
  89
  90         with open(self.template_root + template_path, "r") as myfile:
  91             data = myfile.read()
  92
  93         parsed = pystache.parse(u"%(data)s" % locals())
  94         renderer = pystache.Renderer()
  95
  96         output = renderer.render(parsed, variables)
  97
  98         with open(os.getcwd() + "/temp/" + output_path, "w") as myfile:
  99             myfile.write(output)
 100         return os.getcwd() + "/temp/" + output_path
 101
 102
 103 #
 104 # The array_str method takes an array of strings and formats it into a
 105 #  string such that it can be used in an akka configuration file
 106 #
 107 def array_str(arr):
 108     s = "["
 109     for x in range(0, len(arr)):
 110         s = s + '"' + arr[x] + '"'
 111         if x < (len(arr) - 1):
 112             s += ","
 113     s += "]"
 114     return s
 115
 116
 117 #
 118 # The Deployer deploys the controller to one host and configures it
 119 #
 120 class Deployer:
 121     def __init__(self, host, member_no, template, user, password, rootdir,
 122                  distribution, dir_name, hosts, ds_seed_nodes, rpc_seed_nodes,
 123                  replicas, clean=False):
 124         self.host = host
 125         self.member_no = member_no
 126         self.template = template
 127         self.user = user
 128         self.password = password
 129         self.rootdir = rootdir
 130         self.clean = clean
 131         self.distribution = distribution
 132         self.dir_name = dir_name
 133         self.hosts = hosts
 134         self.ds_seed_nodes = ds_seed_nodes
 135         self.rpc_seed_nodes = rpc_seed_nodes
 136         self.replicas = replicas
 137
 138         # Connect to the remote host and start doing operations
 139         self.remote = RemoteHost(self.host, self.user, self.password,
 140                                  self.rootdir)
 141
 142     def kill_controller(self):
 143         self.remote.copy_file("kill_controller.sh", self.rootdir + "/")
 144         self.remote.exec_cmd(self.rootdir + "/kill_controller.sh")
 145
 146     def deploy(self):
 147         # Determine distribution version
 148         distribution_name \
 149             = os.path.splitext(os.path.basename(self.distribution))[0]
 150         distribution_ver = re.search('(\d+\.\d+\.\d+-\w+\Z)|'
 151                                      '(\d+\.\d+\.\d+-\w+)(-RC\d+\Z)|'
 152                                      '(\d+\.\d+\.\d+-\w+)(-RC\d+(\.\d+)\Z)|'
 153                                      '(\d+\.\d+\.\d+-\w+)(-SR\d+\Z)|'
 154                                      '(\d+\.\d+\.\d+-\w+)(-SR\d+(\.\d+)\Z)',
 155                                      distribution_name)  # noqa
 156
 157         if distribution_ver is None:
 158             print distribution_name + " is not a valid distribution version." \
 159                                       " (Must contain version in the form: " \
 160                                       "\"<#>.<#>.<#>-<name>\" or \"<#>.<#>." \
 161                                       "<#>-<name>-SR<#>\" or \"<#>.<#>.<#>" \
 162                                       "-<name>-RC<#>\", e.g. 0.2.0-SNAPSHOT)"
 163             sys.exit(1)
 164         distribution_ver = distribution_ver.group()
 165
 166         # Render all the templates
 167         renderer = TemplateRenderer(self.template)
 168         akka_conf = renderer.render(
 169             "akka.conf.template", "akka.conf",
 170             {
 171                 "HOST": self.host,
 172                 "MEMBER_NAME": "member-" + str(self.member_no),
 173                 "DS_SEED_NODES": array_str(self.ds_seed_nodes),
 174                 "RPC_SEED_NODES": array_str(self.rpc_seed_nodes)
 175             })
 176         module_shards_conf = renderer.render("module-shards.conf.template",
 177                                              "module-shards.conf",
 178                                              self.replicas)
 179         modules_conf = renderer.render("modules.conf.template",
 180                                        "modules.conf")
 181         features_cfg = \
 182             renderer.render("org.apache.karaf.features.cfg.template",
 183                             "org.apache.karaf.features.cfg",
 184                             {"ODL_DISTRIBUTION": distribution_ver})
 185         jolokia_xml = renderer.render("jolokia.xml.template", "jolokia.xml")
 186         management_cfg = \
 187             renderer.render("org.apache.karaf.management.cfg.template",
 188                             "org.apache.karaf.management.cfg",
 189                             {"HOST": self.host})
 190         datastore_cfg = \
 191             renderer.render(
 192                 "org.opendaylight.controller.cluster.datastore.cfg.template",
 193                 "org.opendaylight.controller.cluster.datastore.cfg")
 194
 195         # Delete all the sub-directories under the deploy directory if
 196         # the --clean flag is used
 197         if self.clean is True:
 198             self.remote.exec_cmd("rm -rf " + self.rootdir + "/deploy/*")
 199
 200         # Create the deployment directory
 201         self.remote.mkdir(self.dir_name)
 202
 203         # Clean the m2 repository
 204         self.remote.exec_cmd("rm -rf " + self.rootdir + "/.m2/repository")
 205
 206         # Copy the distribution to the host and unzip it
 207         odl_file_path = self.dir_name + "/odl.zip"
 208         self.remote.copy_file(self.distribution, odl_file_path)
 209         self.remote.exec_cmd("unzip -o " + odl_file_path + " -d " +
 210                              self.dir_name + "/")
 211
 212         # Rename the distribution directory to odl
 213         self.remote.exec_cmd("mv " + self.dir_name + "/" +
 214                              distribution_name + " " + self.dir_name + "/odl")
 215
 216         # Copy all the generated files to the server
 217         self.remote.mkdir(self.dir_name +
 218                           "/odl/configuration/initial")
 219         self.remote.copy_file(akka_conf, self.dir_name +
 220                               "/odl/configuration/initial/")
 221         self.remote.copy_file(module_shards_conf, self.dir_name +
 222                               "/odl/configuration/initial/")
 223         self.remote.copy_file(modules_conf, self.dir_name +
 224                               "/odl/configuration/initial/")
 225         self.remote.copy_file(features_cfg, self.dir_name +
 226                               "/odl/etc/")
 227         self.remote.copy_file(jolokia_xml, self.dir_name +
 228                               "/odl/deploy/")
 229         self.remote.copy_file(management_cfg, self.dir_name +
 230                               "/odl/etc/")
 231
 232         if datastore_cfg is not None:
 233             self.remote.copy_file(datastore_cfg, self.dir_name + "/odl/etc/")
 234
 235         # Add symlink
 236         self.remote.exec_cmd("ln -sfn " + self.dir_name + " " +
 237                              args.rootdir + "/deploy/current")
 238
 239         # Run karaf
 240         self.remote.start_controller(self.dir_name)
 241
 242
 243 def main():
 244     # Validate some input
 245     if os.path.exists(args.distribution) is False:
 246         print args.distribution + " is not a valid file"
 247         sys.exit(1)
 248
 249     if os.path.exists(os.getcwd() + "/templates/" + args.template) is False:
 250         print args.template + " is not a valid template"
 251
 252     # Prepare some 'global' variables
 253     hosts = args.hosts.split(",")
 254     time_stamp = time.time()
 255     dir_name = args.rootdir + "/deploy/" + str(time_stamp)
 256
 257     ds_seed_nodes = []
 258     rpc_seed_nodes = []
 259     all_replicas = []
 260     replicas = {}
 261
 262     for x in range(0, len(hosts)):
 263         ds_seed_nodes.append("akka.tcp://opendaylight-cluster-data@" +
 264                              hosts[x] + ":2550")
 265         rpc_seed_nodes.append("akka.tcp://odl-cluster-rpc@" +
 266                               hosts[x] + ":2551")
 267         all_replicas.append("member-" + str(x + 1))
 268
 269     for x in range(0, 10):
 270         if len(all_replicas) > args.rf:
 271             replicas["REPLICAS_" + str(x + 1)] \
 272                 = array_str(random.sample(all_replicas, args.rf))
 273         else:
 274             replicas["REPLICAS_" + str(x + 1)] = array_str(all_replicas)
 275
 276     deployers = []
 277
 278     for x in range(0, len(hosts)):
 279         deployers.append(Deployer(hosts[x], x + 1, args.template, args.user,
 280                                   args.password, args.rootdir,
 281                                   args.distribution, dir_name, hosts,
 282                                   ds_seed_nodes, rpc_seed_nodes, replicas,
 283                                   args.clean))
 284
 285     for x in range(0, len(hosts)):
 286         deployers[x].kill_controller()
 287
 288     for x in range(0, len(hosts)):
 289         deployers[x].deploy()
 290
 291 # Run the script
 292 main()