test/tools/clustering/cluster-monitor/monitor.py

   1 #!/usr/bin/python
   2 """
   3 Cluster Monitor Tool
   4 Author: Phillip Shea
   5 Updated: 2015-May-07
   6
   7 This tool provides real-time visualization of the cluster member roles for all
   8 shards in the config datastore. It is assumed that all cluster members have the
   9 same shards.
  10
  11 A file named 'cluster.json' contaning a list of the IP addresses of the
  12 controllers is required. This resides in the same directory as monitor.py.
  13 "user" and "pass" are not required for monitor.py, but they may be
  14 needed for other apps in this folder. The file should look like this:
  15
  16     {
  17         "cluster": {
  18             "controllers": [
  19                 "172.17.10.93",
  20                 "172.17.10.94",
  21                 "172.17.10.95"
  22             ],
  23             "user": "username",
  24             "pass": "password"
  25         }
  26     }
  27
  28 Usage:python monitor.py
  29 """
  30 from io import BytesIO
  31 import time
  32 import pprint
  33 import curses
  34 import sys
  35 import json
  36 import pycurl
  37 import string
  38
  39
  40 def rest_get(restURL):
  41     rest_buffer = BytesIO()
  42     c = pycurl.Curl()
  43     c.setopt(c.TIMEOUT, 2)
  44     c.setopt(c.CONNECTTIMEOUT, 1)
  45     c.setopt(c.FAILONERROR, False)
  46     c.setopt(c.URL, str(restURL))
  47     c.setopt(c.HTTPGET, 0)
  48     c.setopt(c.WRITEFUNCTION, rest_buffer.write)
  49     c.perform()
  50     c.close()
  51     return json.loads(rest_buffer.getvalue())
  52
  53
  54 def getClusterRolesWithCurl(shardName, *args):
  55     ips = args[0]
  56     names = args[1]
  57     controller_state = {}
  58     for i, ip in enumerate(ips):
  59         controller_state[ip] = None
  60         url = 'http://' + ip + ':' + '8181/jolokia/read/org.opendaylight.controller:'
  61         url += 'Category=Shards,name=' + names[i]
  62         url += '-shard-' + shardName + '-config,type=DistributedConfigDatastore'
  63         try:
  64             resp = rest_get(url)
  65             if resp['status'] != 200:
  66                 controller_state[ip] = 'HTTP ' + str(resp['status'])
  67             if 'value' in resp:
  68                 data_value = resp['value']
  69                 controller_state[ip] = data_value['RaftState']
  70         except:
  71             if 'timed out' in str(sys.exc_info()[1]):
  72                 controller_state[ip] = 'timeout'
  73             elif 'JSON' in str(sys.exc_info()):
  74                 controller_state[ip] = 'JSON error'
  75             elif 'connect to host' in str(sys.exc_info()):
  76                 controller_state[ip] = 'no connection'
  77             else:
  78                 controller_state[ip] = 'down'
  79     return controller_state
  80
  81
  82 def size_and_color(cluster_roles, field_length, ip_addr):
  83     status_dict = {}
  84     status_dict['txt'] = string.center(str(cluster_roles[ip_addr]), field_length)
  85     if cluster_roles[ip_addr] == "Leader":
  86         status_dict['color'] = curses.color_pair(2)
  87     elif cluster_roles[ip_addr] == "Follower":
  88         status_dict['color'] = curses.color_pair(3)
  89     elif cluster_roles[ip_addr] == "Candidate":
  90         status_dict['color'] = curses.color_pair(5)
  91     else:
  92         status_dict['color'] = curses.color_pair(0)
  93     return status_dict
  94
  95
  96 field_len = 14
  97 try:
  98     with open('cluster.json') as cluster_file:
  99         data = json.load(cluster_file)
 100 except:
 101     print str(sys.exc_info())
 102     print 'Unable to open the file cluster.json'
 103     exit(1)
 104 try:
 105     controllers = data["cluster"]["controllers"]
 106 except:
 107     print str(sys.exc_info())
 108     print 'Error reading the file cluster.json'
 109     exit(1)
 110
 111 controller_names = []
 112 # Retrieve controller names and shard names.
 113 for controller in controllers:
 114     url = "http://" + controller + ":8181/jolokia/read/org.opendaylight.controller:"
 115     url += "Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore"
 116     try:
 117         data = rest_get(url)
 118     except:
 119         print 'Unable to retrieve shard names from ' + controller
 120         print 'Are all controllers up?'
 121         print str(sys.exc_info()[1])
 122         exit(1)
 123     print 'shards from the first controller'
 124     pprint.pprint(data)
 125     # grab the controller name from the first shard
 126     name = data['value']['LocalShards'][0]
 127     print name
 128     pos = name.find('-shard-')
 129     print pos
 130     print name[:8]
 131     controller_names.append(name[:name.find('-shard-')])
 132 print controller_names
 133 # Putting shard names in a list, assuming all controllers have the same shards.
 134 Shards = data['value']['LocalShards']
 135 for i, shard in enumerate(Shards):
 136     Shards[i] = Shards[i].replace('member-', '')
 137     Shards[i] = Shards[i].replace('-shard-', '')
 138     Shards[i] = Shards[i].replace('-config', '')
 139     Shards[i] = Shards[i].replace(Shards[i][0], '')
 140 print Shards
 141
 142 stdscr = curses.initscr()
 143 curses.noecho()
 144 curses.cbreak()
 145 curses.curs_set(0)
 146 stdscr.keypad(1)
 147 stdscr.nodelay(1)
 148
 149 curses.start_color()
 150 curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)
 151 curses.init_pair(2, curses.COLOR_WHITE, curses.COLOR_GREEN)
 152 curses.init_pair(3, curses.COLOR_WHITE, curses.COLOR_BLUE)
 153 curses.init_pair(4, curses.COLOR_WHITE, curses.COLOR_YELLOW)
 154 curses.init_pair(5, curses.COLOR_BLACK, curses.COLOR_YELLOW)
 155
 156 # display controller and shard headers
 157 for data_column, controller in enumerate(controller_names):
 158     stdscr.addstr(0, field_len * (data_column + 1), string.center(controller, field_len), curses.color_pair(1))
 159 for row, shard in enumerate(Shards):
 160     stdscr.addstr(row + 1, 0, shard, curses.color_pair(1))
 161 stdscr.addstr(len(Shards) + 2, 0, 'Press q to quit.', curses.color_pair(1))
 162 stdscr.refresh()
 163
 164 # display shard status
 165 odd_or_even = 0
 166 key = ''
 167 while key != ord('q') and key != ord('Q'):
 168     odd_or_even += 1
 169     key = stdscr.getch()
 170
 171     for row, shard_name in enumerate(Shards):
 172         cluster_stat = getClusterRolesWithCurl(shard_name, controllers, controller_names)
 173         for data_column, controller in enumerate(controllers):
 174             status = size_and_color(cluster_stat, field_len, controller)
 175             stdscr.addstr(row + 1, field_len * (data_column + 1), status['txt'], status['color'])
 176     time.sleep(0.5)
 177     if odd_or_even % 2 == 0:
 178         stdscr.addstr(0, field_len/2 - 2, " <3 ", curses.color_pair(5))
 179     else:
 180         stdscr.addstr(0, field_len/2 - 2, " <3 ", curses.color_pair(0))
 181     stdscr.refresh()
 182
 183 # clean up
 184 curses.nocbreak()
 185 stdscr.keypad(0)
 186 curses.echo()
 187 curses.endwin()