X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Fscripts%2Fgenerate_visState.py;h=f8d1cd269f66e13892f508d7e2cb9e18c3136e42;hb=f5be5440e8ef95263193ef4d26958198722e4e44;hp=093cd1a0dbccc09379c6d7a16589749f455cb06c;hpb=e8652b63fac0ee8b68ac599f8994707d71e81187;p=integration%2Ftest.git

diff --git a/csit/scripts/generate_visState.py b/csit/scripts/generate_visState.py
index 093cd1a0db..f8d1cd269f 100644
--- a/csit/scripts/generate_visState.py
+++ b/csit/scripts/generate_visState.py
@@ -12,68 +12,70 @@ from copy import deepcopy as dc
 
 import json
 
+# Pretty Printer
+
 
 def p(x):
     print(json.dumps(x, indent=4, sort_keys=True))
 
 
 class visState:
+    # viState template
     def __init__(self):
         self.content = {
-            'title': None,
-            'type': None,
-            'params': {
-                'type': None,
-                'grid': {
-                    'categoryLines': False,
-                    'style': {
-                        'color': '#eee'
-                    }
-                },
-                'categoryAxes': None,
-                'valueAxes': None,
-                'seriesParams': None,
-                'addTooltip': True,
-                'addLegend': True,
-                'legendPosition': 'right',
-                'times': [],
-                'addTimeMarker': False
+            "title": None,
+            "type": None,
+            "params": {
+                "type": None,
+                "grid": {"categoryLines": False, "style": {"color": "#eee"}},
+                "categoryAxes": None,
+                "valueAxes": None,
+                "seriesParams": None,
+                "addTooltip": True,
+                "addLegend": True,
+                "legendPosition": "right",
+                "times": [],
+                "addTimeMarker": False,
             },
-            'aggs': None
+            "aggs": None,
         }
 
     def create(self, config):
         temp = self.content
-        temp['title'] = config['title']
-        temp['type'] = temp['params']['type'] = config['type']
+        temp["title"] = config["title"]
+        temp["type"] = temp["params"]["type"] = config["type"]
 
         cat = categoryAxes()
-        temp['params']['categoryAxes'] = [dc(
-            cat.create()) for i in range(config['num_cat_axes'])]
+        temp["params"]["categoryAxes"] = [
+            dc(cat.create()) for i in range(config["num_cat_axes"])
+        ]
 
         val = ValueAxes()
-        temp['params']['valueAxes'] = [dc(val.create(position=i['position'],
-                                                     title=i['title']))
-                                       for _, i in
-                                       config['value_axes'].items()]
+        temp["params"]["valueAxes"] = [
+            dc(val.create(position=i["position"], title=i["title"]))
+            for _, i in config["value_axes"].items()
+        ]
 
         agg = aggs()
 
-        temp['aggs'] = \
-            [dc(agg.create(id=i,
-                           field=config['aggs'][i]['field'],
-                           custom_label=config['aggs'][i]['custom_label'],
-                           schema=config['aggs'][i]['schema']))
-             for i in range(1, len(config['aggs']) + 1)]
-
-        temp['params']['seriesParams'] = [seriesParams(i['data_type'],
-                                                       i['mode'],
-                                                       i['label'],
-                                                       i['agg_id'],
-                                                       i['value_axis'])
-                                          .create()
-                                          for _, i in
-                                          config['seriesParams'].items()]
+        temp["aggs"] = [
+            dc(
+                agg.create(
+                    id=i,
+                    field=config["aggs"][i]["field"],
+                    custom_label=config["aggs"][i]["custom_label"],
+                    schema=config["aggs"][i]["schema"],
+                )
+            )
+            for i in range(1, len(config["aggs"]) + 1)
+        ]
+
+        temp["params"]["seriesParams"] = [
+            seriesParams(
+                i["data_type"], i["mode"], i["label"], i["agg_id"], i["value_axis"]
+            ).create()
+            for _, i in config["seriesParams"].items()
+        ]
 
         return temp
 
@@ -81,122 +83,129 @@ class visState:
 class categoryAxes:
     def __init__(self):
         self.content = {
-            'id': None,
-            'type': 'category',
-            'position': 'bottom',
-            'show': True,
-            'style': {},
-            'scale': {
-                'type': 'linear'
-            },
-            'labels': {
-                'show': True,
-                'truncate': 100
-            },
-            'title': {}
+            "id": None,
+            "type": "category",
+            "position": "bottom",
+            "show": True,
+            "style": {},
+            "scale": {"type": "linear"},
+            "labels": {"show": True, "truncate": 100},
+            "title": {},
         }
         self.counter = 0
 
+    # Category axes are named as CategoryAxis-i
     def create(self):
         self.counter += 1
         temp = dc(self.content)
-        temp['id'] = 'CategoryAxis-{}'.format(self.counter)
+        temp["id"] = "CategoryAxis-{}".format(self.counter)
         return temp
 
 
 class ValueAxes:
     def __init__(self):
         self.content = {
-            'id': None,
-            'name': None,
-            'type': 'value',
-            'position': 'left',
-            'show': True,
-            'style': {},
-            'scale': {
-                'type': 'linear',
-                'mode': 'normal'
-            },
-            'labels': {
-                'show': True,
-                'rotate': 0,
-                'filter': False,
-                'truncate': 100
-            },
-            'title': {
-                'text': None
-            }
+            "id": None,
+            "name": None,
+            "type": "value",
+            "position": "left",
+            "show": True,
+            "style": {},
+            "scale": {"type": "linear", "mode": "normal"},
+            "labels": {"show": True, "rotate": 0, "filter": False, "truncate": 100},
+            "title": {"text": None},
         }
         self.counter = 0
 
-    def create(self, position='left', title='Value'):
+    def create(self, position="left", title="Value"):
         self.counter += 1
         temp = dc(self.content)
-        temp['id'] = 'ValueAxis-{}'.format(self.counter)
-        if position == 'left':
-            temp['name'] = 'LeftAxis-{}'.format(self.counter)
-        elif position == 'right':
-            temp['name'] = 'RightAxis-{}'.format(self.counter)
+        temp["id"] = "ValueAxis-{}".format(self.counter)
+        if position == "left":
+            temp["name"] = "LeftAxis-{}".format(self.counter)
+        elif position == "right":
+            temp["name"] = "RightAxis-{}".format(self.counter)
         else:
             # raise ValueError('Not one of left or right')
             # assuming default
-            temp['name'] = 'LeftAxis-{}'.format(self.counter)
+            temp["name"] = "LeftAxis-{}".format(self.counter)
 
-        temp['title']['text'] = title
+        temp["title"]["text"] = title
 
         return temp
 
 
+# 'seriesParams' are the ones that actually show up in the plots.
+# They point to a data source a.k.a 'aggs' (short for aggregation)
+# to get their data.
+
+
 class seriesParams:
     def __init__(self, data_type, mode, label, agg_id, value_axis):
         self.content = {
-            'show': True,
-            'type': data_type,
-            'mode': mode,
-            'data': {
-                'label': label,
-                'id': str(agg_id)
+            "show": True,
+            "type": data_type,
+            "mode": mode,
+            "data": {
+                "label": label,
+                "id": str(agg_id),  # the id of the aggregation they point to
             },
-            'valueAxis': 'ValueAxis-{}'.format(value_axis),
-            'drawLinesBetweenPoints': True,
-            'showCircles': True
+            "valueAxis": "ValueAxis-{}".format(value_axis),
+            "drawLinesBetweenPoints": True,
+            "showCircles": True,
         }
 
     def create(self):
         return self.content
 
 
+# 'aggs' or aggregation refers to collection of values. They are the data
+# source which are used by seriesParams. and as expected they take 'field'
+# as the nested name of the key.
+#
+# Example, if your value is in {
+#  'perfomance': {
+#  'plots': {
+#         'rate': myval,
+#          ...
+#        }
+#   },
+#   then I would have to use, 'performance.plots.rate' as the 'field' for aggs
+# the 'schema' of an agg is 'metric' which are to be
+# plotted in the Y-axis and 'segment' for the ones in X-axis
+
+
 class aggs:
     def __init__(self):
         self.content = {
-            'id': None,
-            'enabled': True,
-            'type': None,
-            'schema': None,
-            'params': {
-                'field': None,
-                'customLabel': None
-            }
+            "id": None,
+            "enabled": True,
+            "type": None,
+            "schema": None,
+            "params": {"field": None, "customLabel": None},
         }
         self.counter = 0
 
     def create(self, id, field, custom_label, schema):
         temp = dc(self.content)
-        temp['id'] = id
-        temp['params']['field'] = field
-        temp['params']['customLabel'] = custom_label
-        temp['schema'] = schema
-        if schema == 'metric':
-            temp['type'] = 'max'
+        temp["id"] = id
+        temp["params"]["field"] = field
+        temp["params"]["customLabel"] = custom_label
+        temp["schema"] = schema
+        if schema == "metric":
+            temp["type"] = "max"
             return temp
-        elif schema == 'segment':
-            temp['type'] = 'terms'
-            temp['params']['size'] = 20  # default
-            temp['params']['order'] = 'asc'
-            temp['params']['orderBy'] = '_term'
+        elif schema == "segment":
+            temp["type"] = "terms"
+            temp["params"]["size"] = 20  # default
+            temp["params"]["order"] = "asc"
+            temp["params"]["orderBy"] = "_term"
         return temp
 
 
+# 'series' actually combines and simplifies both 'seriesParams' and 'aggs'
+# Both 'seriesParams' and 'aggs' support 'default' to set default values
+
 # generate takes both the template config and project specific config and
 # parses and organizes as much info available from that and
 # generates an intermediate format first which
@@ -204,6 +213,7 @@ class aggs:
 # be sent to Kibana. Hence, any error occuring in the visualizaton side
 # must first be checked by looking at the intermediate format.
 
+
 def generate(dash_config, viz_config):
 
     format = {
@@ -215,15 +225,10 @@ def generate(dash_config, viz_config):
         "id": None,
         "aggs": {},
         "title": None,
-        "num_cat_axes": None
+        "num_cat_axes": None,
     }
 
-    value_axes_format = {
-        "index": {
-            "position": None,
-            "title": None
-        }
-    }
+    value_axes_format = {"index": {"position": None, "title": None}}
 
     seriesParams_format = {
         "index": {
@@ -231,22 +236,22 @@ def generate(dash_config, viz_config):
             "data_type": None,
             "mode": None,
             "label": None,
-            "agg_id": None
+            "agg_id": None,
         }
     }
 
-    aggs_format = {
-        "index": {
-            "custom_label": None,
-            "field": None,
-            "schema": None
-        }
-    }
+    aggs_format = {"index": {"custom_label": None, "field": None, "schema": None}}
 
     # all general description must be present in either of the config files
     for config in [viz_config, dash_config]:
-        general_fields = ['type', 'index_pattern',
-                          'num_cat_axes', 'title', 'desc', 'id']
+        general_fields = [
+            "type",
+            "index_pattern",
+            "num_cat_axes",
+            "title",
+            "desc",
+            "id",
+        ]
         for i in general_fields:
             try:
                 format[i] = config[i]
@@ -254,170 +259,180 @@ def generate(dash_config, viz_config):
                 pass
 
     # setting any default values if available
-    mappings = {'value_axes': value_axes_format,
-                'seriesParams': seriesParams_format, 'aggs': aggs_format}
+    mappings = {
+        "value_axes": value_axes_format,
+        "seriesParams": seriesParams_format,
+        "aggs": aggs_format,
+    }
     for index, container in mappings.items():
         try:
-            default_values = viz_config[index]['default']
+            default_values = viz_config[index]["default"]
             for i in default_values:
-                container['index'][i] = default_values[i]
+                container["index"][i] = default_values[i]
         except Exception:
             pass
 
+    ####################################################################
+    # Extract 'value_axes', 'seriesParams' or 'aggs' if present in viz_config
     value_axes_counter = 1
-    for m in viz_config['value_axes']:
+    for m in viz_config["value_axes"]:
         if m != "default":
             temp = dc(value_axes_format)
-            temp[str(value_axes_counter)] = temp['index']
-            for i in ['position', 'title']:
+            temp[str(value_axes_counter)] = temp["index"]
+            for i in ["position", "title"]:
                 try:
-                    temp[str(value_axes_counter)
-                         ][i] = viz_config['value_axes'][m][i]
+                    temp[str(value_axes_counter)][i] = viz_config["value_axes"][m][i]
                 except KeyError:
                     pass
-            format['value_axes'].update(temp)
+            format["value_axes"].update(temp)
             value_axes_counter += 1
 
-    seriesParams_fields = ['value_axis',
-                           'data_type', 'mode', 'label', 'agg_id']
+    seriesParams_fields = ["value_axis", "data_type", "mode", "label", "agg_id"]
     try:
-        for m in viz_config['seriesParams']:
-            if m != 'default':
+        for m in viz_config["seriesParams"]:
+            if m != "default":
                 temp = dc(seriesParams_format)
-                temp[m] = temp['index']
+                temp[m] = temp["index"]
                 for i in seriesParams_fields:
                     try:
-                        temp[m][i] = viz_config['seriesParams'][m][i]
+                        temp[m][i] = viz_config["seriesParams"][m][i]
                     except KeyError:
                         pass
-                format['seriesParams'].update(temp)
+                format["seriesParams"].update(temp)
     except KeyError:
         pass
 
     agg_counter = 1
     try:
-        for m in viz_config['aggs']:
-            if m != 'default':
+        for m in viz_config["aggs"]:
+            if m != "default":
                 temp = dc(aggs_format)
-                temp[m] = temp['index']
-                for i in ['field', 'custom_label', 'schema']:
+                temp[m] = temp["index"]
+                for i in ["field", "custom_label", "schema"]:
                     try:
-                        temp[m][i] = viz_config['aggs'][m][i]
+                        temp[m][i] = viz_config["aggs"][m][i]
                     except KeyError:
                         pass
-                format['aggs'].update(temp)
+                format["aggs"].update(temp)
     except KeyError:
         pass
+    ####################################################################
 
+    # collect 'series' from both the configs
     configs = []
     try:
-        viz_config['series']
+        viz_config["series"]
         configs.append(viz_config)
     except KeyError:
         pass
 
     try:
-        dash_config['y-axis']['series']
-        configs.append(dash_config['y-axis'])
+        dash_config["y-axis"]["series"]
+        configs.append(dash_config["y-axis"])
     except KeyError:
         pass
 
+    ########################################################################
+    # Extract 'series' from either of the configs
     for config in configs:
         try:
             value_axes_counter = 1
-            for key in config['value_axes']:
+            for key in config["value_axes"]:
 
                 value_axes_temp = dc(value_axes_format)
-                value_axes_temp[str(value_axes_counter)
-                                ] = value_axes_temp['index']
+                value_axes_temp[str(value_axes_counter)] = value_axes_temp["index"]
 
-                for index in ['position', 'title']:
+                for index in ["position", "title"]:
                     try:
-                        value_axes_temp[str(
-                            value_axes_counter)][index] = \
-                            config['value_axes'][key][index]
+                        value_axes_temp[str(value_axes_counter)][index] = config[
+                            "value_axes"
+                        ][key][index]
                     except KeyError as e:
                         pass
-                format['value_axes'].update(value_axes_temp)
+                format["value_axes"].update(value_axes_temp)
                 value_axes_counter += 1
 
         except KeyError as e:
             pass
 
         try:
-            for key in config['series']:
+            for key in config["series"]:
                 try:
                     # check if this key is present or not
-                    config['series'][key]['not_in_seriesParams']
+                    config["series"][key]["not_in_seriesParams"]
                 except KeyError:
                     seriesParams_temp = dc(seriesParams_format)
-                    seriesParams_temp[key] = seriesParams_temp['index']
-                    for index in ['value_axis', 'data_type', 'mode', 'label']:
+                    seriesParams_temp[key] = seriesParams_temp["index"]
+                    for index in ["value_axis", "data_type", "mode", "label"]:
                         try:
-                            seriesParams_temp[key][index] = \
-                                config['series'][key][index]
+                            seriesParams_temp[key][index] = config["series"][key][index]
                         except KeyError as e:
                             pass
-                    seriesParams_temp[key]['agg_id'] = key
-                    format['seriesParams'].update(seriesParams_temp)
+                    seriesParams_temp[key]["agg_id"] = key
+                    format["seriesParams"].update(seriesParams_temp)
                 finally:
                     agg_temp = dc(aggs_format)
-                    agg_temp[key] = agg_temp['index']
-                    for index in ['field', 'schema']:
+                    agg_temp[key] = agg_temp["index"]
+                    for index in ["field", "schema"]:
                         try:
-                            agg_temp[key][index] = config['series'][key][index]
+                            agg_temp[key][index] = config["series"][key][index]
                         except KeyError as e:
                             pass
-                    agg_temp[key]['custom_label'] = \
-                        config['series'][key]['label']
-                    format['aggs'].update(agg_temp)
+                    agg_temp[key]["custom_label"] = config["series"][key]["label"]
+                    format["aggs"].update(agg_temp)
         except KeyError as e:
             print("required fields are empty!")
 
+    ##########################################################################
+
     # to remove the default template index
-    for i in ['value_axes', 'seriesParams', 'aggs']:
+    for i in ["value_axes", "seriesParams", "aggs"]:
         try:
-            format[i].pop('index')
+            format[i].pop("index")
         except KeyError:
             # print("No default index found")
             pass
 
-    if not config_validator(format):
-        raise ValueError('Missing required field values')
+    missing = config_validator(format)
+    if len(missing):
+        raise ValueError("Missing required field values :-", *missing)
 
     p(format)
 
     vis = visState()
     generated_visState = vis.create(format)
 
-    # checking incase there are None values \
+    # checking incase there are None values
     # in the format indicating missing fields
 
-    if not config_validator(generated_visState):
-        raise KeyError('required fields are missing values!')
+    missing = config_validator(generated_visState)
+    if len(missing):
+        raise ValueError("required fields are missing values! ", *missing)
     return format, generated_visState
 
 
 # Check the generated format if it contains any key with None
 # as it's value which indicates incomplete information
-def config_validator(val):
-    flag = True
-    for _, i in val.items():
-        if isinstance(i, dict):
-            flag = config_validator(i)
-        if i is None:
-            return False
-    return flag
-
-
-if __name__ == '__main__':
-    with open('viz.yaml', 'r') as f:
+
+
+def config_validator(val, missing=[]):
+    for key, value in val.items():
+        if isinstance(value, dict):
+            config_validator(value)
+        if value is None:
+            missing.append(key)
+    return missing
+
+
+if __name__ == "__main__":
+    with open("viz_config.yaml", "r") as f:
         viz_config = yaml.safe_load(f)
 
-    with open('dashboard.yaml', 'r') as f:
+    with open("dash_config.yaml", "r") as f:
         dash_config = yaml.safe_load(f)
 
-    generate(dash_config['dashboard']['viz'][2],
-             viz_config['opendaylight-test-performance'])
+    generate(
+        dash_config["dashboard"]["viz"][2], viz_config["opendaylight-test-performance"]
+    )
     # generate(dash_config['dashboard']['viz'][3],viz_config['opendaylight-test-performance'])
     # generate(dash_config['dashboard']['viz'][1],viz_config['opendaylight-test-feature'])