csit/scripts/generate_visState.py

   1 # SPDX-License-Identifier: EPL-1.0
   2 ##############################################################################
   3 # Copyright (c) 2018 The Linux Foundation and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Eclipse Public License v1.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.eclipse.org/legal/epl-v10.html
   9 ##############################################################################
  10 import yaml
  11 import copy
  12
  13 import json
  14
  15
  16 dc = copy.deepcopy
  17
  18
  19 def p(x):
  20     print(json.dumps(x, indent=4, sort_keys=True))
  21
  22
  23 class visState:
  24     def __init__(self):
  25         self.content = {
  26             'title': None,
  27             'type': None,
  28             'params': {
  29                 'type': None,
  30                 'grid': {
  31                     'categoryLines': False,
  32                     'style': {
  33                         'color': '#eee'
  34                     }
  35                 },
  36                 'categoryAxes': None,
  37                 'valueAxes': None,
  38                 'seriesParams': None,
  39                 'addTooltip': True,
  40                 'addLegend': True,
  41                 'legendPosition': 'right',
  42                 'times': [],
  43                 'addTimeMarker': False
  44             },
  45             'aggs': None
  46         }
  47
  48     def create(self, config):
  49         temp = self.content
  50         temp['title'] = config['title']
  51         temp['type'] = temp['params']['type'] = config['type']
  52
  53         cat = categoryAxes()
  54         temp['params']['categoryAxes'] = [dc(
  55             cat.create()) for i in range(config['num_cat_axes'])]
  56
  57         val = ValueAxes()
  58         temp['params']['valueAxes'] = [dc(val.create(position=i['position'],
  59                                                      title=i['title']))
  60                                        for _, i in
  61                                        config['value_axes'].items()]
  62
  63         agg = aggs()
  64
  65         temp['aggs'] = [dc(agg.create(field=i['field'],
  66                                       custom_label=i['custom_label'],
  67                                       schema=i['schema']))
  68                         for _, i in
  69                         config['aggs'].items()]
  70
  71         temp['params']['seriesParams'] = [seriesParams(i['data_type'],
  72                                                        i['mode'],
  73                                                        i['label'],
  74                                                        i['agg_id'],
  75                                                        i['value_axis'])
  76                                           .create()
  77                                           for _, i in
  78                                           config['seriesParams'].items()]
  79
  80         return temp
  81
  82
  83 class categoryAxes:
  84     def __init__(self):
  85         self.content = {
  86             'id': None,
  87             'type': 'category',
  88             'position': 'bottom',
  89             'show': True,
  90             'style': {},
  91             'scale': {
  92                 'type': 'linear'
  93             },
  94             'labels': {
  95                 'show': True,
  96                 'truncate': 100
  97             },
  98             'title': {}
  99         }
 100         self.counter = 0
 101
 102     def create(self):
 103         self.counter += 1
 104         temp = dc(self.content)
 105         temp['id'] = 'CategoryAxis-{}'.format(self.counter)
 106         return temp
 107
 108
 109 class ValueAxes:
 110     def __init__(self):
 111         self.content = {
 112             'id': None,
 113             'name': None,
 114             'type': 'value',
 115             'position': 'left',
 116             'show': True,
 117             'style': {},
 118             'scale': {
 119                 'type': 'linear',
 120                 'mode': 'normal'
 121             },
 122             'labels': {
 123                 'show': True,
 124                 'rotate': 0,
 125                 'filter': False,
 126                 'truncate': 100
 127             },
 128             'title': {
 129                 'text': None
 130             }
 131         }
 132         self.counter = 0
 133
 134     def create(self, position='left', title='Value'):
 135         self.counter += 1
 136         temp = dc(self.content)
 137         temp['id'] = 'ValueAxis-{}'.format(self.counter)
 138         if position == 'left':
 139             temp['name'] = 'LeftAxis-{}'.format(self.counter)
 140         elif position == 'right':
 141             temp['name'] = 'RightAxis-{}'.format(self.counter)
 142         else:
 143             # raise ValueError('Not one of left or right')
 144             # assuming default
 145             temp['name'] = 'LeftAxis-{}'.format(self.counter)
 146
 147         temp['title']['text'] = title
 148
 149         return temp
 150
 151
 152 class seriesParams:
 153     def __init__(self, data_type, mode, label, agg_id, value_axis):
 154         self.content = {
 155             'show': True,
 156             'type': data_type,
 157             'mode': mode,
 158             'data': {
 159                 'label': label,
 160                 'id': str(agg_id)
 161             },
 162             'valueAxis': 'ValueAxis-{}'.format(value_axis),
 163             'drawLinesBetweenPoints': True,
 164             'showCircles': True
 165         }
 166
 167     def create(self):
 168         return self.content
 169
 170
 171 class aggs:
 172     def __init__(self):
 173         self.content = {
 174             'id': None,
 175             'enabled': True,
 176             'type': None,
 177             'schema': None,
 178             'params': {
 179                 'field': None,
 180                 'customLabel': None
 181             }
 182         }
 183         self.counter = 0
 184
 185     def create(self, field, custom_label, schema):
 186         self.counter += 1
 187         temp = dc(self.content)
 188         temp['id'] = str(self.counter)
 189         temp['params']['field'] = field
 190         temp['params']['customLabel'] = custom_label
 191         temp['schema'] = schema
 192         if schema == 'metric':
 193             temp['type'] = 'max'
 194             return temp
 195         elif schema == 'segment':
 196             temp['type'] = 'terms'
 197             temp['params']['size'] = 20  # default
 198             temp['params']['order'] = 'asc'
 199             temp['params']['orderBy'] = '_term'
 200         return temp
 201
 202
 203 # generate takes both the template config and project specific config and
 204 # parses and organizes as much info available from that and
 205 # generates an intermediate format first which
 206 # contains all necessary info to deterministically create the visState to
 207 # be sent to Kibana. Hence, any error occuring in the visualizaton side
 208 # must first be checked by looking at the intermediate format.
 209
 210 def generate(dash_config, viz_config):
 211
 212     format = {
 213         "type": None,
 214         "value_axes": {},
 215         "seriesParams": {},
 216         "index_pattern": None,
 217         "desc": None,
 218         "id": None,
 219         "aggs": {},
 220         "title": None,
 221         "num_cat_axes": None
 222     }
 223
 224     value_axes_format = {
 225         "index": {
 226             "position": None,
 227             "title": None
 228         }
 229     }
 230
 231     seriesParams_format = {
 232         "index": {
 233             "value_axis": None,
 234             "data_type": None,
 235             "mode": None,
 236             "label": None,
 237             "agg_id": None
 238         }
 239     }
 240
 241     aggs_format = {
 242         "index": {
 243             "custom_label": None,
 244             "field": None,
 245             "schema": None
 246         }
 247     }
 248
 249     # all general description must be present in either of the config files
 250     for config in [viz_config, dash_config]:
 251         general_fields = ['type', 'index_pattern',
 252                           'num_cat_axes', 'title', 'desc', 'id']
 253         for i in general_fields:
 254             try:
 255                 format[i] = config[i]
 256             except KeyError as e:
 257                 pass
 258
 259     # setting any default values if available
 260     mappings = {'value_axes': value_axes_format,
 261                 'seriesParams': seriesParams_format, 'aggs': aggs_format}
 262     for index, container in mappings.items():
 263         try:
 264             default_values = viz_config[index]['default']
 265             for i in default_values:
 266                 container['index'][i] = default_values[i]
 267         except Exception:
 268             pass
 269
 270     value_axes_counter = 1
 271     for m in viz_config['value_axes']:
 272         if m != "default":
 273             temp = dc(value_axes_format)
 274             temp[str(value_axes_counter)] = temp['index']
 275             for i in ['position', 'title']:
 276                 try:
 277                     temp[str(value_axes_counter)
 278                          ][i] = viz_config['value_axes'][m][i]
 279                 except KeyError:
 280                     pass
 281             format['value_axes'].update(temp)
 282             value_axes_counter += 1
 283
 284     seriesParams_counter = 1
 285     seriesParams_fields = ['value_axis',
 286                            'data_type', 'mode', 'label', 'agg_id']
 287     try:
 288         for m in viz_config['seriesParams']:
 289             if m != 'default':
 290                 temp = dc(seriesParams_format)
 291                 temp[str(seriesParams_counter)] = temp['index']
 292                 for i in seriesParams_fields:
 293                     try:
 294                         temp[str(seriesParams_counter)
 295                              ][i] = viz_config['seriesParams'][m][i]
 296                     except KeyError:
 297                         pass
 298                 format['seriesParams'].update(temp)
 299                 seriesParams_counter += 1
 300     except KeyError:
 301         pass
 302
 303     agg_counter = 1
 304     try:
 305         for m in viz_config['aggs']:
 306             if m != 'default':
 307                 temp = dc(aggs_format)
 308                 temp[str(agg_counter)] = temp['index']
 309                 for i in ['field', 'custom_label', 'schema']:
 310                     try:
 311                         temp[str(agg_counter)][i] = viz_config['aggs'][m][i]
 312                     except KeyError:
 313                         pass
 314                 format['aggs'].update(temp)
 315                 agg_counter += 1
 316     except KeyError:
 317         pass
 318
 319     configs = []
 320     try:
 321         viz_config['series']
 322         configs.append(viz_config)
 323     except KeyError:
 324         pass
 325
 326     try:
 327         dash_config['y-axis']['series']
 328         configs.append(dash_config['y-axis'])
 329     except KeyError:
 330         pass
 331
 332     for config in configs:
 333         try:
 334             value_axes_counter = 1
 335             for key in config['value_axes']:
 336
 337                 value_axes_temp = dc(value_axes_format)
 338                 value_axes_temp[str(value_axes_counter)
 339                                 ] = value_axes_temp['index']
 340
 341                 for index in ['position', 'title']:
 342                     try:
 343                         value_axes_temp[str(
 344                             value_axes_counter)][index] = \
 345                             config['value_axes'][key][index]
 346                     except KeyError as e:
 347                         pass
 348                 format['value_axes'].update(value_axes_temp)
 349                 value_axes_counter += 1
 350
 351         except KeyError as e:
 352             pass
 353
 354         try:
 355             for key in config['series']:
 356                 try:
 357                     # check if this key is present or not
 358                     config['series'][key]['not_in_seriesParams']
 359                 except KeyError:
 360                     seriesParams_temp = dc(seriesParams_format)
 361                     seriesParams_temp[str(
 362                         seriesParams_counter)] = seriesParams_temp['index']
 363                     for index in ['value_axis', 'data_type', 'mode', 'label']:
 364                         try:
 365                             seriesParams_temp[str(
 366                                 seriesParams_counter)][index] = \
 367                                 config['series'][key][index]
 368                         except KeyError as e:
 369                             pass
 370                     seriesParams_temp[str(
 371                         seriesParams_counter)]['agg_id'] = agg_counter
 372                     format['seriesParams'].update(seriesParams_temp)
 373                     seriesParams_counter += 1
 374                 finally:
 375                     agg_temp = dc(aggs_format)
 376                     agg_temp[str(agg_counter)] = agg_temp['index']
 377                     for index in ['field', 'schema']:
 378                         try:
 379                             agg_temp[str(agg_counter)
 380                                      ][index] = config['series'][key][index]
 381                         except KeyError as e:
 382                             pass
 383                     agg_temp[str(
 384                         agg_counter)]['custom_label'] = \
 385                         config['series'][key]['label']
 386                     format['aggs'].update(agg_temp)
 387                     agg_counter += 1
 388         except KeyError as e:
 389             print("required fields are empty!")
 390
 391     # to remove the default template index
 392     for i in ['value_axes', 'seriesParams', 'aggs']:
 393         try:
 394             format[i].pop('index')
 395         except KeyError:
 396             # print("No default index found")
 397             pass
 398
 399     if not config_validator(format):
 400         raise ValueError('Missing required field values')
 401
 402     p(format)
 403
 404     vis = visState()
 405     generated_visState = vis.create(format)
 406
 407     # checking incase there are None values \
 408     # in the format indicating missing fields
 409
 410     if not config_validator(generated_visState):
 411         raise KeyError('required fields are missing values!')
 412     return format, generated_visState
 413
 414
 415 # Check the generated format if it contains any key with None
 416 # as it's value which indicates incomplete information
 417 def config_validator(val):
 418     flag = True
 419     for _, i in val.items():
 420         if isinstance(i, dict):
 421             flag = config_validator(i)
 422         if i is None:
 423             return False
 424     return flag
 425
 426
 427 if __name__ == '__main__':
 428     with open('viz.yaml', 'r') as f:
 429         viz_config = yaml.safe_load(f)
 430
 431     with open('dashboard.yaml', 'r') as f:
 432         dash_config = yaml.safe_load(f)
 433
 434     generate(dash_config['dashboard']['viz'][2],
 435              viz_config['opendaylight-test-performance'])
 436     # generate(dash_config['dashboard']['viz'][3],viz_config['opendaylight-test-performance'])
 437     # generate(dash_config['dashboard']['viz'][1],viz_config['opendaylight-test-feature'])