2 Appenders Object Definition to be used with karaf-decanter
3 Used to collect resource usage metrics
5 Currently implements ElasticsearchAppender
8 declare foo = ElasticsearchAppender()
9 get connection object conn = foo.get_connection(host=host,
12 foo.get_jvm_memory(conn), foo.get_jvm_classloading(conn),
13 foo.get_jvm_threadingconn(),
14 foo.get_jvm_garbagecollector(conn),
15 foo.get_jvm_operatingsystem(conn)
17 the latest resource usage statistics dictionary object
18 (latest based on the @timestamp)
20 foo.plot_points(conn, title, filename, metric,
21 submetric, submetrickey)
24 foo.plot_points(conn, 'JVM Started Threads',
25 'threadcount.png', 'Threading',
26 'TotalStartedThreadCount')
27 submetrickey is optional
28 for more usage and examples see https://goo.gl/dT1RqT
32 from datetime import datetime
33 from operator import itemgetter
34 from elasticsearch import Elasticsearch
35 from elasticsearch_dsl import Search
37 import matplotlib as mpl
41 class MBeanNotFoundError(Exception):
42 def __init__(self, message, errors):
43 super(MBeanNotFoundError, self).__init__(message)
46 class BaseAppender(object):
48 Base Appender from which all appenders should inherit
51 def _get_index(self, need_all):
52 raise NotImplementedError
54 def _get_connection(self):
55 raise NotImplementedError
58 class ElasticsearchAppender(BaseAppender):
60 ElasticsearchAppender Class
61 Metrics supported : Memory, ClassLoading, Threading, GarbageCollector
62 Individual resource attributes as defined in attr dictionary object
65 attr = {'Memory': ['HeapMemoryUsage', 'NonHeapMemoryUsage',
67 'ClassLoading': ['TotalLoadedClassCount', 'UnloadedClassCount',
69 'OperatingSystem': ['FreeSwapSpaceSize', 'TotalSwapSpaceSize',
70 'FreePhysicalMemorySize',
71 'TotalPhysicalMemorySize',
72 'CommittedVirtualMemorySize', 'ProcessCpuLoad',
73 'ProcessCpuTime', 'SystemCpuLoad',
75 'Threading': ['DaemonThreadCount', 'PeakThreadCount',
76 'ThreadCount', 'TotalStartedThreadCount',
78 'GarbageCollector': ['LastGcInfo', 'CollectionCount',
79 '@timestamp', 'CollectionTime']}
80 label = {'Memory': 'Memory', 'ClassLoading': 'Class Loading',
81 'Threading': 'Threads', 'GarbageCollector': 'Garbage Collector'}
83 def get_connection(self, host='localhost', port=9200):
84 host = self.cleanse_string(host)
85 port = self.cleanse_string(port)
86 return self._get_connection(host, port)
88 def get_jvm_memory(self, connection):
89 return self._get_mbean_attr(connection, 'Memory')
91 def get_jvm_classloading(self, connection):
92 return self._get_mbean_attr(connection, 'ClassLoading',)
94 def get_jvm_threading(self, connection):
95 return self._get_mbean_attr(connection, 'Threading')
97 def get_jvm_garbagecollector(self, connection):
98 return self._get_mbean_attr(connection, 'GarbageCollector')
100 def get_jvm_operatingsystem(self, connection):
101 return self._get_mbean_attr(connection, 'OperatingSystem')
103 def cleanse_string(self, s):
104 return str(s).replace("'", "")
106 def plot_points(self, connection, title, filename, metric, submetric,
109 from matplotlib import dates, pyplot as plt, ticker as tkr
111 metric = self.cleanse_string(metric)
112 submetric = self.cleanse_string(submetric)
113 if submetrickey is not None:
114 submetrickey = self.cleanse_string(submetrickey)
116 points = self._get_plot_points(connection, metric, submetric,
118 points[0] = [p.replace(microsecond=0) for p in points[0]]
119 myFmt = dates.DateFormatter('%m-%d %H:%M:%S')
120 fig, ax = plt.subplots()
122 ax.plot_date(points[0], points[1], 'c-')
123 ax.grid(color='grey')
124 ax.patch.set_facecolor('black')
125 ax.xaxis.set_major_formatter(myFmt)
128 axes.get_yaxis().get_major_formatter().set_scientific(False)
129 axes.get_yaxis().get_major_formatter().set_useOffset(False)
131 ax.set_xlabel('Time')
132 xlabel = self._convert(submetric).title()
133 if submetrickey is not None:
134 xlabel = xlabel + ' : ' + str(submetrickey).title()
135 ax.set_ylabel(xlabel)
137 mx = max(points[1]) + max(points[1]) * 0.00001
138 mn = min(points[1]) - min(points[1]) * 0.00001
142 if isinstance(points[1][0], int):
143 axes.yaxis.set_major_formatter(tkr.FuncFormatter(lambda x, _:
146 axes.yaxis.set_major_formatter(tkr.FuncFormatter(lambda x, _:
148 plt.gcf().autofmt_xdate()
149 plt.savefig(filename, bbox_inches='tight')
151 def _convert(self, name):
152 s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', name)
153 return re.sub('([a-z0-9])([A-Z])', r'\1 \2', s1).lower()
155 def _get_y_val(self, response, metric, submetric=None):
156 if isinstance(response[metric], dict):
157 return response[metric][submetric]
159 return response[metric]
161 def _get_plot_points(self, connection, metric, submetric,
163 indices = self._get_index(connection, need_all=True)
165 for index in indices:
166 responses = self._get_all_mbean_attr(connection, metric, index)
167 for response in responses:
168 point = (self._get_datetime_object(response['@timestamp']),
169 self._get_y_val(response, submetric, submetrickey))
171 points.sort(key=itemgetter(0))
174 def _get_index(self, connection, need_all=False):
175 indices = sorted([i for i in
176 connection.indices.get_mapping().keys()
177 if i.startswith('karaf')])
181 return sorted(indices, reverse=True)[0]
183 def _get_connection(self, host, port):
184 con_obj = {'host': host, 'port': port}
185 es = Elasticsearch([con_obj])
188 def _get_all_mbean_attr(self, connection, mbean, index, dsl_class='match'):
189 s = Search(using=connection, index=index).\
190 query(dsl_class, ObjectName=mbean).\
191 sort({"@timestamp": {"order": 'desc'}})
194 response.append(self._get_attr_obj([hit], mbean))
197 def _get_mbean_attr(self, connection, mbean, dsl_class='match'):
198 index = self._get_index(connection)
201 s = Search(using=connection, index=index).\
202 query(dsl_class, ObjectName=mbean).\
203 sort({"@timestamp": {"order": 'desc'}})[0].execute()
205 raise MBeanNotFoundError('Could Not Fetch %s mbean' % mbean)
207 mem_attr = self._get_attr_obj(s, mbean)
210 def _get_attr_obj(self, response, mbean):
213 for k in self.attr[mbean]:
214 is_to_dict = getattr(r[k], "to_dict", None)
215 if callable(is_to_dict):
216 mbean_attr[k] = r[k].to_dict()
221 def _get_datetime_object(self, timestamp):
222 return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S,%fZ')