# SPDX-License-Identifier: EPL-1.0 ############################################################################## # Copyright (c) 2018 The Linux Foundation and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Eclipse Public License v1.0 # which accompanies this distribution, and is available at # http://www.eclipse.org/legal/epl-v10.html ############################################################################## import yaml from copy import deepcopy as dc import json # Pretty Printer def p(x): print(json.dumps(x, indent=4, sort_keys=True)) class visState: # viState template def __init__(self): self.content = { 'title': None, 'type': None, 'params': { 'type': None, 'grid': { 'categoryLines': False, 'style': { 'color': '#eee' } }, 'categoryAxes': None, 'valueAxes': None, 'seriesParams': None, 'addTooltip': True, 'addLegend': True, 'legendPosition': 'right', 'times': [], 'addTimeMarker': False }, 'aggs': None } def create(self, config): temp = self.content temp['title'] = config['title'] temp['type'] = temp['params']['type'] = config['type'] cat = categoryAxes() temp['params']['categoryAxes'] = [dc( cat.create()) for i in range(config['num_cat_axes'])] val = ValueAxes() temp['params']['valueAxes'] = [dc(val.create(position=i['position'], title=i['title'])) for _, i in config['value_axes'].items()] agg = aggs() temp['aggs'] = \ [dc(agg.create(id=i, field=config['aggs'][i]['field'], custom_label=config['aggs'][i]['custom_label'], schema=config['aggs'][i]['schema'])) for i in range(1, len(config['aggs']) + 1)] temp['params']['seriesParams'] = [seriesParams(i['data_type'], i['mode'], i['label'], i['agg_id'], i['value_axis']) .create() for _, i in config['seriesParams'].items()] return temp class categoryAxes: def __init__(self): self.content = { 'id': None, 'type': 'category', 'position': 'bottom', 'show': True, 'style': {}, 'scale': { 'type': 'linear' }, 'labels': { 'show': True, 'truncate': 100 }, 'title': {} } self.counter = 0 # Category axes are named as CategoryAxis-i def create(self): self.counter += 1 temp = dc(self.content) temp['id'] = 'CategoryAxis-{}'.format(self.counter) return temp class ValueAxes: def __init__(self): self.content = { 'id': None, 'name': None, 'type': 'value', 'position': 'left', 'show': True, 'style': {}, 'scale': { 'type': 'linear', 'mode': 'normal' }, 'labels': { 'show': True, 'rotate': 0, 'filter': False, 'truncate': 100 }, 'title': { 'text': None } } self.counter = 0 def create(self, position='left', title='Value'): self.counter += 1 temp = dc(self.content) temp['id'] = 'ValueAxis-{}'.format(self.counter) if position == 'left': temp['name'] = 'LeftAxis-{}'.format(self.counter) elif position == 'right': temp['name'] = 'RightAxis-{}'.format(self.counter) else: # raise ValueError('Not one of left or right') # assuming default temp['name'] = 'LeftAxis-{}'.format(self.counter) temp['title']['text'] = title return temp # 'seriesParams' are the ones that actually show up in the plots. # They point to a data source a.k.a 'aggs' (short for aggregation) # to get their data. class seriesParams: def __init__(self, data_type, mode, label, agg_id, value_axis): self.content = { 'show': True, 'type': data_type, 'mode': mode, 'data': { 'label': label, 'id': str(agg_id) # the id of the aggregation they point to }, 'valueAxis': 'ValueAxis-{}'.format(value_axis), 'drawLinesBetweenPoints': True, 'showCircles': True } def create(self): return self.content # 'aggs' or aggregation refers to collection of values. They are the data # source which are used by seriesParams. and as expected they take 'field' # as the nested name of the key. # # Example, if your value is in { # 'perfomance': { # 'plots': { # 'rate': myval, # ... # } # }, # then I would have to use, 'performance.plots.rate' as the 'field' for aggs # the 'schema' of an agg is 'metric' which are to be # plotted in the Y-axis and 'segment' for the ones in X-axis class aggs: def __init__(self): self.content = { 'id': None, 'enabled': True, 'type': None, 'schema': None, 'params': { 'field': None, 'customLabel': None } } self.counter = 0 def create(self, id, field, custom_label, schema): temp = dc(self.content) temp['id'] = id temp['params']['field'] = field temp['params']['customLabel'] = custom_label temp['schema'] = schema if schema == 'metric': temp['type'] = 'max' return temp elif schema == 'segment': temp['type'] = 'terms' temp['params']['size'] = 20 # default temp['params']['order'] = 'asc' temp['params']['orderBy'] = '_term' return temp # 'series' actually combines and simplifies both 'seriesParams' and 'aggs' # Both 'seriesParams' and 'aggs' support 'default' to set default values # generate takes both the template config and project specific config and # parses and organizes as much info available from that and # generates an intermediate format first which # contains all necessary info to deterministically create the visState to # be sent to Kibana. Hence, any error occuring in the visualizaton side # must first be checked by looking at the intermediate format. def generate(dash_config, viz_config): format = { "type": None, "value_axes": {}, "seriesParams": {}, "index_pattern": None, "desc": None, "id": None, "aggs": {}, "title": None, "num_cat_axes": None } value_axes_format = { "index": { "position": None, "title": None } } seriesParams_format = { "index": { "value_axis": None, "data_type": None, "mode": None, "label": None, "agg_id": None } } aggs_format = { "index": { "custom_label": None, "field": None, "schema": None } } # all general description must be present in either of the config files for config in [viz_config, dash_config]: general_fields = ['type', 'index_pattern', 'num_cat_axes', 'title', 'desc', 'id'] for i in general_fields: try: format[i] = config[i] except KeyError as e: pass # setting any default values if available mappings = {'value_axes': value_axes_format, 'seriesParams': seriesParams_format, 'aggs': aggs_format} for index, container in mappings.items(): try: default_values = viz_config[index]['default'] for i in default_values: container['index'][i] = default_values[i] except Exception: pass #################################################################### # Extract 'value_axes', 'seriesParams' or 'aggs' if present in viz_config value_axes_counter = 1 for m in viz_config['value_axes']: if m != "default": temp = dc(value_axes_format) temp[str(value_axes_counter)] = temp['index'] for i in ['position', 'title']: try: temp[str(value_axes_counter) ][i] = viz_config['value_axes'][m][i] except KeyError: pass format['value_axes'].update(temp) value_axes_counter += 1 seriesParams_fields = ['value_axis', 'data_type', 'mode', 'label', 'agg_id'] try: for m in viz_config['seriesParams']: if m != 'default': temp = dc(seriesParams_format) temp[m] = temp['index'] for i in seriesParams_fields: try: temp[m][i] = viz_config['seriesParams'][m][i] except KeyError: pass format['seriesParams'].update(temp) except KeyError: pass agg_counter = 1 try: for m in viz_config['aggs']: if m != 'default': temp = dc(aggs_format) temp[m] = temp['index'] for i in ['field', 'custom_label', 'schema']: try: temp[m][i] = viz_config['aggs'][m][i] except KeyError: pass format['aggs'].update(temp) except KeyError: pass #################################################################### # collect 'series' from both the configs configs = [] try: viz_config['series'] configs.append(viz_config) except KeyError: pass try: dash_config['y-axis']['series'] configs.append(dash_config['y-axis']) except KeyError: pass ######################################################################## # Extract 'series' from either of the configs for config in configs: try: value_axes_counter = 1 for key in config['value_axes']: value_axes_temp = dc(value_axes_format) value_axes_temp[str(value_axes_counter) ] = value_axes_temp['index'] for index in ['position', 'title']: try: value_axes_temp[str( value_axes_counter)][index] = \ config['value_axes'][key][index] except KeyError as e: pass format['value_axes'].update(value_axes_temp) value_axes_counter += 1 except KeyError as e: pass try: for key in config['series']: try: # check if this key is present or not config['series'][key]['not_in_seriesParams'] except KeyError: seriesParams_temp = dc(seriesParams_format) seriesParams_temp[key] = seriesParams_temp['index'] for index in ['value_axis', 'data_type', 'mode', 'label']: try: seriesParams_temp[key][index] = \ config['series'][key][index] except KeyError as e: pass seriesParams_temp[key]['agg_id'] = key format['seriesParams'].update(seriesParams_temp) finally: agg_temp = dc(aggs_format) agg_temp[key] = agg_temp['index'] for index in ['field', 'schema']: try: agg_temp[key][index] = config['series'][key][index] except KeyError as e: pass agg_temp[key]['custom_label'] = \ config['series'][key]['label'] format['aggs'].update(agg_temp) except KeyError as e: print("required fields are empty!") ########################################################################## # to remove the default template index for i in ['value_axes', 'seriesParams', 'aggs']: try: format[i].pop('index') except KeyError: # print("No default index found") pass missing = config_validator(format) if len(missing): raise ValueError('Missing required field values :-', *missing) p(format) vis = visState() generated_visState = vis.create(format) # checking incase there are None values # in the format indicating missing fields missing = config_validator(generated_visState) if len(missing): raise ValueError('required fields are missing values! ', *missing) return format, generated_visState # Check the generated format if it contains any key with None # as it's value which indicates incomplete information def config_validator(val, missing=[]): for key, value in val.items(): if isinstance(value, dict): config_validator(value) if value is None: missing.append(key) return missing if __name__ == '__main__': with open('viz_config.yaml', 'r') as f: viz_config = yaml.safe_load(f) with open('dash_config.yaml', 'r') as f: dash_config = yaml.safe_load(f) generate(dash_config['dashboard']['viz'][2], viz_config['opendaylight-test-performance']) # generate(dash_config['dashboard']['viz'][3],viz_config['opendaylight-test-performance']) # generate(dash_config['dashboard']['viz'][1],viz_config['opendaylight-test-feature'])