X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FAppenders%2FElasticsearchAppender.py;h=efb1a6b663f9f7ee33389130b756b379c6f7c219;hb=1c106006ffb33c1c60e2f8abe61e88a7bd95a7bd;hp=c7c67d05182e4dd1458222384ca07cbc1d197e4d;hpb=8d9fde3e0d522eb4714fbbee642054560143b948;p=integration%2Ftest.git diff --git a/csit/libraries/Appenders/ElasticsearchAppender.py b/csit/libraries/Appenders/ElasticsearchAppender.py index c7c67d0518..efb1a6b663 100644 --- a/csit/libraries/Appenders/ElasticsearchAppender.py +++ b/csit/libraries/Appenders/ElasticsearchAppender.py @@ -11,7 +11,7 @@ call foo.get_jvm_memory(conn), foo.get_jvm_classloading(conn), foo.get_jvm_threadingconn(), - foo.get_jvm_garbageCollector(conn), + foo.get_jvm_garbagecollector(conn), foo.get_jvm_operatingsystem(conn) returns the latest resource usage statistics dictionary object @@ -35,7 +35,8 @@ from elasticsearch import Elasticsearch from elasticsearch_dsl import Search import re import matplotlib as mpl -mpl.use('Agg') + +mpl.use("Agg") class MBeanNotFoundError(Exception): @@ -44,9 +45,9 @@ class MBeanNotFoundError(Exception): class BaseAppender(object): - ''' + """ Base Appender from which all appenders should inherit - ''' + """ def _get_index(self, need_all): raise NotImplementedError @@ -56,55 +57,73 @@ class BaseAppender(object): class ElasticsearchAppender(BaseAppender): - ''' + """ ElasticsearchAppender Class Metrics supported : Memory, ClassLoading, Threading, GarbageCollector Individual resource attributes as defined in attr dictionary object - ''' - - attr = {'Memory': ['HeapMemoryUsage', 'NonHeapMemoryUsage', - '@timestamp'], - 'ClassLoading': ['TotalLoadedClassCount', 'UnloadedClassCount', - '@timestamp'], - 'OperatingSystem': ['FreeSwapSpaceSize', 'TotalSwapSpaceSize', - 'FreePhysicalMemorySize', - 'TotalPhysicalMemorySize', - 'CommittedVirtualMemorySize', 'ProcessCpuLoad', - 'ProcessCpuTime', 'SystemCpuLoad', - '@timestamp'], - 'Threading': ['DaemonThreadCount', 'PeakThreadCount', - 'ThreadCount', 'TotalStartedThreadCount', - '@timestamp'], - 'GarbageCollector': ['LastGcInfo', 'CollectionCount', - '@timestamp', 'CollectionTime']} - label = {'Memory': 'Memory', 'ClassLoading': 'Class Loading', - 'Threading': 'Threads', 'GarbageCollector': 'Garbage Collector'} - - def get_connection(self, host='localhost', port=9200): + """ + + attr = { + "Memory": ["HeapMemoryUsage", "NonHeapMemoryUsage", "@timestamp"], + "ClassLoading": ["TotalLoadedClassCount", "UnloadedClassCount", "@timestamp"], + "OperatingSystem": [ + "FreeSwapSpaceSize", + "TotalSwapSpaceSize", + "FreePhysicalMemorySize", + "TotalPhysicalMemorySize", + "CommittedVirtualMemorySize", + "ProcessCpuLoad", + "ProcessCpuTime", + "SystemCpuLoad", + "@timestamp", + ], + "Threading": [ + "DaemonThreadCount", + "PeakThreadCount", + "ThreadCount", + "TotalStartedThreadCount", + "@timestamp", + ], + "GarbageCollector": [ + "LastGcInfo", + "CollectionCount", + "@timestamp", + "CollectionTime", + ], + } + label = { + "Memory": "Memory", + "ClassLoading": "Class Loading", + "Threading": "Threads", + "GarbageCollector": "Garbage Collector", + } + + def get_connection(self, host="localhost", port=9200): host = self.cleanse_string(host) port = self.cleanse_string(port) return self._get_connection(host, port) def get_jvm_memory(self, connection): - return self._get_mbean_attr(connection, 'Memory') + return self._get_mbean_attr(connection, "Memory") def get_jvm_classloading(self, connection): - return self._get_mbean_attr(connection, 'ClassLoading',) + return self._get_mbean_attr(connection, "ClassLoading") def get_jvm_threading(self, connection): - return self._get_mbean_attr(connection, 'Threading') + return self._get_mbean_attr(connection, "Threading") def get_jvm_garbagecollector(self, connection): - return self._get_mbean_attr(connection, 'GarbageCollector') + return self._get_mbean_attr(connection, "GarbageCollector") def get_jvm_operatingsystem(self, connection): - return self._get_mbean_attr(connection, 'OperatingSystem') + return self._get_mbean_attr(connection, "OperatingSystem") def cleanse_string(self, s): return str(s).replace("'", "") - def plot_points(self, connection, title, filename, metric, submetric, - submetrickey=None): + def plot_points( + self, connection, title, filename, metric, submetric, submetrickey=None + ): from matplotlib import dates, pyplot as plt, ticker as tkr @@ -113,25 +132,24 @@ class ElasticsearchAppender(BaseAppender): if submetrickey is not None: submetrickey = self.cleanse_string(submetrickey) - points = self._get_plot_points(connection, metric, submetric, - submetrickey) + points = self._get_plot_points(connection, metric, submetric, submetrickey) points[0] = [p.replace(microsecond=0) for p in points[0]] - myFmt = dates.DateFormatter('%m-%d %H:%M:%S') + myFmt = dates.DateFormatter("%m-%d %H:%M:%S") fig, ax = plt.subplots() - ax.plot_date(points[0], points[1], 'c-') - ax.grid(color='grey') - ax.patch.set_facecolor('black') + ax.plot_date(points[0], points[1], "c-") + ax.grid(color="grey") + ax.patch.set_facecolor("black") ax.xaxis.set_major_formatter(myFmt) axes = plt.gca() axes.get_yaxis().get_major_formatter().set_scientific(False) axes.get_yaxis().get_major_formatter().set_useOffset(False) - ax.set_xlabel('Time') + ax.set_xlabel("Time") xlabel = self._convert(submetric).title() if submetrickey is not None: - xlabel = xlabel + ' : ' + str(submetrickey).title() + xlabel = xlabel + " : " + str(submetrickey).title() ax.set_ylabel(xlabel) mx = max(points[1]) + max(points[1]) * 0.00001 @@ -140,17 +158,15 @@ class ElasticsearchAppender(BaseAppender): ax.set_title(title) if isinstance(points[1][0], int): - axes.yaxis.set_major_formatter(tkr.FuncFormatter(lambda x, _: - int(x))) + axes.yaxis.set_major_formatter(tkr.FuncFormatter(lambda x, _: int(x))) else: - axes.yaxis.set_major_formatter(tkr.FuncFormatter(lambda x, _: - float(x))) + axes.yaxis.set_major_formatter(tkr.FuncFormatter(lambda x, _: float(x))) plt.gcf().autofmt_xdate() - plt.savefig(filename, bbox_inches='tight') + plt.savefig(filename, bbox_inches="tight") def _convert(self, name): - s1 = re.sub('(.)([A-Z][a-z]+)', r'\1 \2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1 \2', s1).lower() + s1 = re.sub("(.)([A-Z][a-z]+)", r"\1 \2", name) + return re.sub("([a-z0-9])([A-Z])", r"\1 \2", s1).lower() def _get_y_val(self, response, metric, submetric=None): if isinstance(response[metric], dict): @@ -158,51 +174,61 @@ class ElasticsearchAppender(BaseAppender): else: return response[metric] - def _get_plot_points(self, connection, metric, submetric, - submetrickey=None): + def _get_plot_points(self, connection, metric, submetric, submetrickey=None): indices = self._get_index(connection, need_all=True) points = [] for index in indices: responses = self._get_all_mbean_attr(connection, metric, index) for response in responses: - point = (self._get_datetime_object(response['@timestamp']), - self._get_y_val(response, submetric, submetrickey)) + point = ( + self._get_datetime_object(response["@timestamp"]), + self._get_y_val(response, submetric, submetrickey), + ) points.append(point) points.sort(key=itemgetter(0)) return zip(*points) def _get_index(self, connection, need_all=False): - indices = sorted([i for i in - connection.indices.get_mapping().keys() - if i.startswith('karaf')]) + indices = sorted( + [ + i + for i in connection.indices.get_mapping().keys() + if i.startswith("karaf") + ] + ) if need_all: return indices else: return sorted(indices, reverse=True)[0] def _get_connection(self, host, port): - con_obj = {'host': host, 'port': port} + con_obj = {"host": host, "port": port} es = Elasticsearch([con_obj]) return es - def _get_all_mbean_attr(self, connection, mbean, index, dsl_class='match'): - s = Search(using=connection, index=index).\ - query(dsl_class, ObjectName=mbean).\ - sort({"@timestamp": {"order": 'desc'}}) + def _get_all_mbean_attr(self, connection, mbean, index, dsl_class="match"): + s = ( + Search(using=connection, index=index) + .query(dsl_class, ObjectName=mbean) + .sort({"@timestamp": {"order": "desc"}}) + ) response = [] for hit in s.scan(): response.append(self._get_attr_obj([hit], mbean)) return response - def _get_mbean_attr(self, connection, mbean, dsl_class='match'): + def _get_mbean_attr(self, connection, mbean, dsl_class="match"): index = self._get_index(connection) try: - s = Search(using=connection, index=index).\ - query(dsl_class, ObjectName=mbean).\ - sort({"@timestamp": {"order": 'desc'}})[0].execute() + s = ( + Search(using=connection, index=index) + .query(dsl_class, ObjectName=mbean) + .sort({"@timestamp": {"order": "desc"}})[0] + .execute() + ) except Exception: - raise MBeanNotFoundError('Could Not Fetch %s mbean' % mbean) + raise MBeanNotFoundError("Could Not Fetch %s mbean" % mbean) mem_attr = self._get_attr_obj(s, mbean) return mem_attr @@ -219,4 +245,4 @@ class ElasticsearchAppender(BaseAppender): return mbean_attr def _get_datetime_object(self, timestamp): - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S,%fZ') + return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S,%fZ")