tools/clustering/cluster-deployer/deploy.py

   1 #!/usr/local/bin/python
   2 #
   3 # This script deploys a cluster
   4 # -------------------------------------------
   5 #
   6 # Pre-requisites
   7 # - Python 2.7
   8 # - SSHLibrary (pip install robotframework-sshlibrary)
   9 # - pystache (pip install pystache)
  10 # - argparse (pip install argparse)
  11 #
  12 # The input that this script will take is as follows,
  13 #
  14 # - A comma separated list of ip addresses/hostnames for each host on which
  15 #   the distribution needs to be deployed
  16 # - The replication factor to be used
  17 # - The ssh username/password of the remote host(s). Note that this should be
  18 #   the same for each host
  19 # - The name of the template to be used.
  20 #   Note that this template name should match the name of a template folder in
  21 #   the templates directory.
  22 #   The templates directory can be found in the same directory as this script.
  23 #
  24 # Here are the things it will do,
  25 #
  26 # - Copy over a distribution of opendaylight to the remote host
  27 # - Create a timestamped directory on the remote host
  28 # - Unzip the distribution to the timestamped directory
  29 # - Copy over the template substituted configuration files to the appropriate
  30 #   location on the remote host
  31 # - Create a symlink to the timestamped directory
  32 # - Start karaf
  33 #
  34 # ----------------------------------------------------------------------------
  35
  36 import argparse
  37 import time
  38 import pystache
  39 import os
  40 import sys
  41 import random
  42 import re
  43 from remote_host import RemoteHost
  44
  45 parser = argparse.ArgumentParser(description='Cluster Deployer')
  46 parser.add_argument("--distribution", default="",
  47                     help="the absolute path of the distribution on the local "
  48                          "host that needs to be deployed. (Must contain "
  49                          "version in the form: \"<#>.<#>.<#>-<name>\", e.g. "
  50                          "0.2.0-SNAPSHOT)",
  51                     required=True)
  52 parser.add_argument("--rootdir", default="/root",
  53                     help="the root directory on the remote host where the "
  54                          "distribution is to be deployed",
  55                     required=True)
  56 parser.add_argument("--hosts", default="", help="a comma separated list of "
  57                                                 "host names or ip addresses",
  58                     required=True)
  59 parser.add_argument("--clean", action="store_true", default=False,
  60                     help="clean the deployment on the remote host")
  61 parser.add_argument("--template", default="openflow",
  62                     help="the name of the template to be used. "
  63                     "This name should match a folder in the templates "
  64                          "directory.")
  65 parser.add_argument("--rf", default=3, type=int,
  66                     help="replication factor. This is the number of replicas "
  67                          "that should be created for each shard.")
  68 parser.add_argument("--user", default="root", help="the SSH username for the "
  69                                                    "remote host(s)")
  70 parser.add_argument("--password", default="Ecp123",
  71                     help="the SSH password for the remote host(s)")
  72 args = parser.parse_args()
  73
  74
  75 #
  76 # The TemplateRenderer provides methods to render a template
  77 #
  78 class TemplateRenderer:
  79     def __init__(self, template):
  80         self.cwd = os.getcwd()
  81         self.template_root = self.cwd + "/templates/" + template + "/"
  82
  83     def render(self, template_path, output_path, variables=None):
  84         if variables is None:
  85             variables = {}
  86
  87         if os.path.exists(self.template_root + template_path) is False:
  88             return
  89
  90         with open(self.template_root + template_path, "r") as myfile:
  91             data = myfile.read()
  92
  93         parsed = pystache.parse(u"%(data)s" % locals())
  94         renderer = pystache.Renderer()
  95
  96         output = renderer.render(parsed, variables)
  97
  98         with open(os.getcwd() + "/temp/" + output_path, "w") as myfile:
  99             myfile.write(output)
 100         return os.getcwd() + "/temp/" + output_path
 101
 102
 103 #
 104 # The array_str method takes an array of strings and formats it into a
 105 #  string such that it can be used in an akka configuration file
 106 #
 107 def array_str(arr):
 108     s = "["
 109     for x in range(0, len(arr)):
 110         s = s + '"' + arr[x] + '"'
 111         if x < (len(arr) - 1):
 112             s += ","
 113     s += "]"
 114     return s
 115
 116
 117 #
 118 # The Deployer deploys the controller to one host and configures it
 119 #
 120 class Deployer:
 121     def __init__(self, host, member_no, template, user, password, rootdir,
 122                  distribution, dir_name, hosts, ds_seed_nodes, rpc_seed_nodes,
 123                  replicas, clean=False):
 124         self.host = host
 125         self.member_no = member_no
 126         self.template = template
 127         self.user = user
 128         self.password = password
 129         self.rootdir = rootdir
 130         self.clean = clean
 131         self.distribution = distribution
 132         self.dir_name = dir_name
 133         self.hosts = hosts
 134         self.ds_seed_nodes = ds_seed_nodes
 135         self.rpc_seed_nodes = rpc_seed_nodes
 136         self.replicas = replicas
 137
 138         # Connect to the remote host and start doing operations
 139         self.remote = RemoteHost(self.host, self.user, self.password,
 140                                  self.rootdir)
 141
 142     def kill_controller(self):
 143         self.remote.copy_file("kill_controller.sh", self.rootdir + "/")
 144         self.remote.exec_cmd(self.rootdir + "/kill_controller.sh")
 145
 146     def deploy(self):
 147         # Determine distribution version
 148         distribution_name \
 149             = os.path.splitext(os.path.basename(self.distribution))[0]
 150         distribution_ver = re.search('(\d+\.\d+\.\d+-\w+\Z)|'
 151                                      '(\d+\.\d+\.\d+-\w+)(-SR\d+\Z)|'
 152                                      '(\d+\.\d+\.\d+-\w+)(-SR\d+(\.\d+)\Z)',
 153                                      distribution_name)  # noqa
 154
 155         if distribution_ver is None:
 156             print distribution_name + " is not a valid distribution version." \
 157                                       " (Must contain version in the form: " \
 158                                       "\"<#>.<#>.<#>-<name>\" or \"<#>.<#>." \
 159                                       "<#>-<name>-SR<#>\" or \"<#>.<#>.<#>" \
 160                                       "-<name>\", e.g. 0.2.0-SNAPSHOT)"  # noqa
 161             sys.exit(1)
 162         distribution_ver = distribution_ver.group()
 163
 164         # Render all the templates
 165         renderer = TemplateRenderer(self.template)
 166         akka_conf = renderer.render(
 167             "akka.conf.template", "akka.conf",
 168             {
 169                 "HOST": self.host,
 170                 "MEMBER_NAME": "member-" + str(self.member_no),
 171                 "DS_SEED_NODES": array_str(self.ds_seed_nodes),
 172                 "RPC_SEED_NODES": array_str(self.rpc_seed_nodes)
 173             })
 174         module_shards_conf = renderer.render("module-shards.conf.template",
 175                                              "module-shards.conf",
 176                                              self.replicas)
 177         modules_conf = renderer.render("modules.conf.template",
 178                                        "modules.conf")
 179         features_cfg = \
 180             renderer.render("org.apache.karaf.features.cfg.template",
 181                             "org.apache.karaf.features.cfg",
 182                             {"ODL_DISTRIBUTION": distribution_ver})
 183         jolokia_xml = renderer.render("jolokia.xml.template", "jolokia.xml")
 184         management_cfg = \
 185             renderer.render("org.apache.karaf.management.cfg.template",
 186                             "org.apache.karaf.management.cfg",
 187                             {"HOST": self.host})
 188         datastore_cfg = \
 189             renderer.render(
 190                 "org.opendaylight.controller.cluster.datastore.cfg.template",
 191                 "org.opendaylight.controller.cluster.datastore.cfg")
 192
 193         # Delete all the sub-directories under the deploy directory if
 194         # the --clean flag is used
 195         if self.clean is True:
 196             self.remote.exec_cmd("rm -rf " + self.rootdir + "/deploy/*")
 197
 198         # Create the deployment directory
 199         self.remote.mkdir(self.dir_name)
 200
 201         # Clean the m2 repository
 202         self.remote.exec_cmd("rm -rf " + self.rootdir + "/.m2/repository")
 203
 204         # Copy the distribution to the host and unzip it
 205         odl_file_path = self.dir_name + "/odl.zip"
 206         self.remote.copy_file(self.distribution, odl_file_path)
 207         self.remote.exec_cmd("unzip -o " + odl_file_path + " -d " +
 208                              self.dir_name + "/")
 209
 210         # Rename the distribution directory to odl
 211         self.remote.exec_cmd("mv " + self.dir_name + "/" +
 212                              distribution_name + " " + self.dir_name + "/odl")
 213
 214         # Copy all the generated files to the server
 215         self.remote.mkdir(self.dir_name +
 216                           "/odl/configuration/initial")
 217         self.remote.copy_file(akka_conf, self.dir_name +
 218                               "/odl/configuration/initial/")
 219         self.remote.copy_file(module_shards_conf, self.dir_name +
 220                               "/odl/configuration/initial/")
 221         self.remote.copy_file(modules_conf, self.dir_name +
 222                               "/odl/configuration/initial/")
 223         self.remote.copy_file(features_cfg, self.dir_name +
 224                               "/odl/etc/")
 225         self.remote.copy_file(jolokia_xml, self.dir_name +
 226                               "/odl/deploy/")
 227         self.remote.copy_file(management_cfg, self.dir_name +
 228                               "/odl/etc/")
 229
 230         if datastore_cfg is not None:
 231             self.remote.copy_file(datastore_cfg, self.dir_name + "/odl/etc/")
 232
 233         # Add symlink
 234         self.remote.exec_cmd("ln -sfn " + self.dir_name + " " +
 235                              args.rootdir + "/deploy/current")
 236
 237         # Run karaf
 238         self.remote.start_controller(self.dir_name)
 239
 240
 241 def main():
 242     # Validate some input
 243     if os.path.exists(args.distribution) is False:
 244         print args.distribution + " is not a valid file"
 245         sys.exit(1)
 246
 247     if os.path.exists(os.getcwd() + "/templates/" + args.template) is False:
 248         print args.template + " is not a valid template"
 249
 250     # Prepare some 'global' variables
 251     hosts = args.hosts.split(",")
 252     time_stamp = time.time()
 253     dir_name = args.rootdir + "/deploy/" + str(time_stamp)
 254
 255     ds_seed_nodes = []
 256     rpc_seed_nodes = []
 257     all_replicas = []
 258     replicas = {}
 259
 260     for x in range(0, len(hosts)):
 261         ds_seed_nodes.append("akka.tcp://opendaylight-cluster-data@" +
 262                              hosts[x] + ":2550")
 263         rpc_seed_nodes.append("akka.tcp://odl-cluster-rpc@" +
 264                               hosts[x] + ":2551")
 265         all_replicas.append("member-" + str(x + 1))
 266
 267     for x in range(0, 10):
 268         if len(all_replicas) > args.rf:
 269             replicas["REPLICAS_" + str(x + 1)] \
 270                 = array_str(random.sample(all_replicas, args.rf))
 271         else:
 272             replicas["REPLICAS_" + str(x + 1)] = array_str(all_replicas)
 273
 274     deployers = []
 275
 276     for x in range(0, len(hosts)):
 277         deployers.append(Deployer(hosts[x], x + 1, args.template, args.user,
 278                                   args.password, args.rootdir,
 279                                   args.distribution, dir_name, hosts,
 280                                   ds_seed_nodes, rpc_seed_nodes, replicas,
 281                                   args.clean))
 282
 283     for x in range(0, len(hosts)):
 284         deployers[x].kill_controller()
 285
 286     for x in range(0, len(hosts)):
 287         deployers[x].deploy()
 288
 289 # Run the script
 290 main()