tools/clustering/cluster-monitor/monitor.py

   1 #!/usr/bin/python
   2 """
   3 Cluster Monitor Tool
   4 Author: Phillip Shea
   5 Updated: 2015-May-07
   6
   7 This tool provides real-time visualization of the cluster member roles for all
   8 shards in the config datastore.
   9
  10 A file named 'cluster.json' contaning a list of the IP addresses of the
  11 controllers is required. This resides in the same directory as monitor.py.
  12 "user" and "pass" are not required for monitor.py, but they may be
  13 needed for other apps in this folder. The file should look like this:
  14
  15     {
  16         "cluster": {
  17             "controllers": [
  18                 "172.17.10.93",
  19                 "172.17.10.94",
  20                 "172.17.10.95"
  21             ],
  22             "user": "username",
  23             "pass": "password"
  24         }
  25     }
  26
  27 Usage:python monitor.py
  28 """
  29 from io import BytesIO
  30 import time
  31 import pprint
  32 import curses
  33 import sys
  34 import json
  35 import pycurl
  36 import string
  37
  38
  39 def rest_get(restURL):
  40     rest_buffer = BytesIO()
  41     c = pycurl.Curl()
  42     c.setopt(c.TIMEOUT, 2)
  43     c.setopt(c.CONNECTTIMEOUT, 1)
  44     c.setopt(c.FAILONERROR, False)
  45     c.setopt(c.URL, str(restURL))
  46     c.setopt(c.HTTPGET, 0)
  47     c.setopt(c.WRITEFUNCTION, rest_buffer.write)
  48     c.perform()
  49     c.close()
  50     return json.loads(rest_buffer.getvalue())
  51
  52
  53 def getClusterRolesWithCurl(shardName, *args):
  54     ips = args[0]
  55     names = args[1]
  56     controller_state = {}
  57     for i, ip in enumerate(ips):
  58         controller_state[ip] = None
  59         url = 'http://' + ip + ':' + '8181/jolokia/read/org.opendaylight.controller:'
  60         url += 'Category=Shards,name=' + names[i]
  61         url += '-shard-' + shardName + '-config,type=DistributedConfigDatastore'
  62         try:
  63             resp = rest_get(url)
  64             if resp['status'] != 200:
  65                 controller_state[ip] = 'HTTP ' + str(resp['status'])
  66             if 'value' in resp:
  67                 data_value = resp['value']
  68                 controller_state[ip] = data_value['RaftState']
  69         except:
  70             if 'timed out' in str(sys.exc_info()[1]):
  71                 controller_state[ip] = 'timeout'
  72             elif 'JSON' in str(sys.exc_info()):
  73                 controller_state[ip] = 'JSON error'
  74             elif 'connect to host' in str(sys.exc_info()):
  75                 controller_state[ip] = 'no connection'
  76             else:
  77                 controller_state[ip] = 'down'
  78     return controller_state
  79
  80
  81 def size_and_color(cluster_roles, field_length, ip_addr):
  82     status_dict = {}
  83     status_dict['txt'] = string.center(str(cluster_roles[ip_addr]), field_length)
  84     if cluster_roles[ip_addr] == "Leader":
  85         status_dict['color'] = curses.color_pair(2)
  86     elif cluster_roles[ip_addr] == "Follower":
  87         status_dict['color'] = curses.color_pair(3)
  88     elif cluster_roles[ip_addr] == "Candidate":
  89         status_dict['color'] = curses.color_pair(5)
  90     else:
  91         status_dict['color'] = curses.color_pair(0)
  92     return status_dict
  93
  94
  95 try:
  96     with open('cluster.json') as cluster_file:
  97         data = json.load(cluster_file)
  98 except:
  99     print str(sys.exc_info())
 100     print 'Unable to open the file cluster.json'
 101     exit(1)
 102 try:
 103     controllers = data["cluster"]["controllers"]
 104 except:
 105     print str(sys.exc_info())
 106     print 'Error reading the file cluster.json'
 107     exit(1)
 108
 109 controller_names = []
 110 Shards = set()
 111 # Retrieve controller names and shard names.
 112 for controller in controllers:
 113     url = "http://" + controller + ":8181/jolokia/read/org.opendaylight.controller:"
 114     url += "Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore"
 115     try:
 116         data = rest_get(url)
 117     except:
 118         print 'Unable to retrieve shard names from ' + controller
 119         print 'Are all controllers up?'
 120         print str(sys.exc_info()[1])
 121         exit(1)
 122     print 'shards from the first controller'
 123     pprint.pprint(data)
 124     # grab the controller name from the first shard
 125     name = data['value']['LocalShards'][0]
 126     print name
 127     pos = name.find('-shard-')
 128     print pos
 129     print name[:8]
 130     controller_names.append(name[:name.find('-shard-')])
 131
 132     # collect shards found in any controller; does not require all controllers to have the same shards
 133     for localShard in data['value']['LocalShards']:
 134         shardName = localShard[(localShard.find("-shard-")+7):localShard.find("-config")]
 135         Shards.add(shardName)
 136 print controller_names
 137 print Shards
 138 field_len = max(map(len, Shards))+2
 139
 140 stdscr = curses.initscr()
 141 curses.noecho()
 142 curses.cbreak()
 143 curses.curs_set(0)
 144 stdscr.keypad(1)
 145 stdscr.nodelay(1)
 146
 147 curses.start_color()
 148 curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)
 149 curses.init_pair(2, curses.COLOR_WHITE, curses.COLOR_GREEN)
 150 curses.init_pair(3, curses.COLOR_WHITE, curses.COLOR_BLUE)
 151 curses.init_pair(4, curses.COLOR_WHITE, curses.COLOR_YELLOW)
 152 curses.init_pair(5, curses.COLOR_BLACK, curses.COLOR_YELLOW)
 153
 154 # display controller and shard headers
 155 for row, controller in enumerate(controller_names):
 156     stdscr.addstr(row + 1, 0, string.center(controller, field_len), curses.color_pair(1))
 157 for data_column, shard in enumerate(Shards):
 158     stdscr.addstr(0, (field_len+1) * (data_column + 1), string.center(shard, field_len), curses.color_pair(1))
 159 stdscr.addstr(len(Shards) + 2, 0, 'Press q to quit.', curses.color_pair(1))
 160 stdscr.refresh()
 161
 162 # display shard status
 163 odd_or_even = 0
 164 key = ''
 165 while key != ord('q') and key != ord('Q'):
 166     odd_or_even += 1
 167     key = stdscr.getch()
 168
 169     for data_column, shard_name in enumerate(Shards):
 170         cluster_stat = getClusterRolesWithCurl(shard_name, controllers, controller_names)
 171         for row, controller in enumerate(controllers):
 172             status = size_and_color(cluster_stat, field_len, controller)
 173             stdscr.addstr(row + 1, (field_len+1) * (data_column + 1), status['txt'], status['color'])
 174     time.sleep(0.5)
 175     if odd_or_even % 2 == 0:
 176         stdscr.addstr(0, field_len/2 - 2, " <3 ", curses.color_pair(5))
 177     else:
 178         stdscr.addstr(0, field_len/2 - 2, " <3 ", curses.color_pair(0))
 179     stdscr.refresh()
 180
 181 # clean up
 182 curses.nocbreak()
 183 stdscr.keypad(0)
 184 curses.echo()
 185 curses.endwin()