tools/clustering/cluster-monitor/monitor.py

   1 #!/usr/bin/python
   2 """
   3 Cluster Monitor Tool
   4 Author: Phillip Shea
   5 Updated: 2016-Mar-07
   6
   7 This tool provides real-time visualization of the cluster member roles for all
   8 shards in the config datastore.
   9
  10 A file named 'cluster.json' contaning a list of the IP addresses and port numbers
  11 of the controllers is required. This resides in the same directory as monitor.py.
  12 "user" and "pass" are not required for monitor.py, but they may be
  13 needed for other apps in this folder. The file should look like this:
  14
  15     {
  16         "cluster": {
  17             "controllers": [
  18                 {"ip": "172.17.10.93", "port": "8181"},
  19                 {"ip": "172.17.10.93", "port": "8181"},
  20                 {"ip": "172.17.10.93", "port": "8181"}
  21             ],
  22             "user": "username",
  23             "pass": "password",
  24             "shards_to_exclude": []  # list of shard names to omit from output
  25         }
  26     }
  27
  28 Usage:python monitor.py
  29 """
  30 from io import BytesIO
  31 import time
  32 import pprint
  33 import curses
  34 import sys
  35 import json
  36 import pycurl
  37 import string
  38
  39
  40 def rest_get(restURL, username, password):
  41     rest_buffer = BytesIO()
  42     c = pycurl.Curl()
  43     c.setopt(c.TIMEOUT, 2)
  44     c.setopt(c.CONNECTTIMEOUT, 1)
  45     c.setopt(c.FAILONERROR, False)
  46     c.setopt(c.URL, str(restURL))
  47     c.setopt(c.HTTPGET, 0)
  48     c.setopt(c.WRITEFUNCTION, rest_buffer.write)
  49     c.setopt(pycurl.USERPWD, "%s:%s" % (str(username), str(password)))
  50     c.perform()
  51     c.close()
  52     return json.loads(rest_buffer.getvalue())
  53
  54
  55 def getClusterRolesWithCurl(shardName, *args):
  56     controllers = args[0]
  57     names = args[1]
  58     controller_state = {}
  59     for i, controller in enumerate(controllers):
  60         controller_state[controller["ip"]] = None
  61         url = "http://" + controller["ip"] + ":" + controller["port"] + "/jolokia/read/org.opendaylight.controller:"
  62         url += 'Category=Shards,name=' + names[i]
  63         url += '-shard-' + shardName + '-config,type=DistributedConfigDatastore'
  64         try:
  65             resp = rest_get(url, username, password)
  66             if resp['status'] != 200:
  67                 controller_state[controller["ip"]] = 'HTTP ' + str(resp['status'])
  68             if 'value' in resp:
  69                 data_value = resp['value']
  70                 controller_state[controller["ip"]] = data_value['RaftState']
  71         except:
  72             if 'timed out' in str(sys.exc_info()[1]):
  73                 controller_state[controller["ip"]] = 'timeout'
  74             elif 'JSON' in str(sys.exc_info()):
  75                 controller_state[controller["ip"]] = 'JSON error'
  76             elif 'connect to host' in str(sys.exc_info()):
  77                 controller_state[controller["ip"]] = 'no connection'
  78             else:
  79                 controller_state[controller["ip"]] = 'down'
  80     return controller_state
  81
  82
  83 def size_and_color(cluster_roles, field_length, ip_addr):
  84     status_dict = {}
  85     status_dict['txt'] = string.center(str(cluster_roles[ip_addr]), field_length)
  86     if cluster_roles[ip_addr] == "Leader":
  87         status_dict['color'] = curses.color_pair(2)
  88     elif cluster_roles[ip_addr] == "Follower":
  89         status_dict['color'] = curses.color_pair(3)
  90     elif cluster_roles[ip_addr] == "Candidate":
  91         status_dict['color'] = curses.color_pair(5)
  92     else:
  93         status_dict['color'] = curses.color_pair(0)
  94     return status_dict
  95
  96
  97 try:
  98     with open('cluster.json') as cluster_file:
  99         data = json.load(cluster_file)
 100 except:
 101     print str(sys.exc_info())
 102     print 'Unable to open the file cluster.json'
 103     exit(1)
 104 try:
 105     controllers = data["cluster"]["controllers"]
 106     shards_to_exclude = data["cluster"]["shards_to_exclude"]
 107     username = data["cluster"]["user"]
 108     password = data["cluster"]["pass"]
 109 except:
 110     print str(sys.exc_info())
 111     print 'Error reading the file cluster.json'
 112     exit(1)
 113
 114 controller_names = []
 115 Shards = set()
 116 # Retrieve controller names and shard names.
 117 for controller in controllers:
 118     url = "http://" + controller["ip"] + ":" + controller["port"] + "/jolokia/read/org.opendaylight.controller:"
 119     url += "Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore"
 120     rest_get(url, username, password)
 121     try:
 122         data = rest_get(url, username, password)
 123     except:
 124         print 'Unable to retrieve shard names from ' + str(controller)
 125         print 'Are all controllers up?'
 126         print str(sys.exc_info()[1])
 127         exit(1)
 128     print 'shards from the first controller'
 129     pprint.pprint(data)
 130     # grab the controller name from the first shard
 131     name = data['value']['LocalShards'][0]
 132     print name
 133     pos = name.find('-shard-')
 134     print pos
 135     print name[:8]
 136     controller_names.append(name[:name.find('-shard-')])
 137
 138     # collect shards found in any controller; does not require all controllers to have the same shards
 139     for localShard in data['value']['LocalShards']:
 140         shardName = localShard[(localShard.find("-shard-") + 7):localShard.find("-config")]
 141         if shardName not in shards_to_exclude:
 142             Shards.add(shardName)
 143 print controller_names
 144 print Shards
 145 field_len = max(map(len, Shards)) + 2
 146
 147 stdscr = curses.initscr()
 148 curses.noecho()
 149 curses.cbreak()
 150 curses.curs_set(0)
 151 stdscr.keypad(1)
 152 stdscr.nodelay(1)
 153
 154 curses.start_color()
 155 curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)
 156 curses.init_pair(2, curses.COLOR_WHITE, curses.COLOR_GREEN)
 157 curses.init_pair(3, curses.COLOR_WHITE, curses.COLOR_BLUE)
 158 curses.init_pair(4, curses.COLOR_WHITE, curses.COLOR_YELLOW)
 159 curses.init_pair(5, curses.COLOR_BLACK, curses.COLOR_YELLOW)
 160
 161 # display controller and shard headers
 162 for row, controller in enumerate(controller_names):
 163     stdscr.addstr(row + 1, 0, string.center(controller, field_len), curses.color_pair(1))
 164 for data_column, shard in enumerate(Shards):
 165     stdscr.addstr(0, (field_len + 1) * (data_column + 1), string.center(shard, field_len), curses.color_pair(1))
 166 stdscr.addstr(len(Shards) + 2, 0, 'Press q to quit.', curses.color_pair(1))
 167 stdscr.refresh()
 168
 169 # display shard status
 170 odd_or_even = 0
 171 key = ''
 172 while key != ord('q') and key != ord('Q'):
 173     odd_or_even += 1
 174     key = stdscr.getch()
 175
 176     for data_column, shard_name in enumerate(Shards):
 177         if shard_name not in shards_to_exclude:
 178             cluster_stat = getClusterRolesWithCurl(shard_name, controllers, controller_names)
 179             for row, controller in enumerate(controllers):
 180                 status = size_and_color(cluster_stat, field_len, controller["ip"])
 181                 stdscr.addstr(row + 1, (field_len + 1) * (data_column + 1), status['txt'], status['color'])
 182         time.sleep(0.5)
 183         if odd_or_even % 2 == 0:
 184             stdscr.addstr(0, field_len / 2 - 2, " <3 ", curses.color_pair(5))
 185         else:
 186             stdscr.addstr(0, field_len / 2 - 2, " <3 ", curses.color_pair(0))
 187         stdscr.refresh()
 188
 189 # clean up
 190 curses.nocbreak()
 191 stdscr.keypad(0)
 192 curses.echo()
 193 curses.endwin()