csit/scripts/push_dashboard.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
   5 ##############################################################################
   6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
   7 #
   8 # All rights reserved. This program and the accompanying materials
   9 # are made available under the terms of the Eclipse Public License v1.0
  10 # which accompanies this distribution, and is available at
  11 # http://www.eclipse.org/legal/epl-v10.html
  12 ##############################################################################
  13
  14 """
  15 This script is used to parse dashboard config files, construct
  16 JSON BODY and push it to ELK DB.
  17
  18 Usage: python push_dashboard.py host:port
  19
  20 """
  21
  22 # stdlib
  23 import json
  24 import os
  25 import sys
  26 import glob
  27
  28 # 3rd party lib
  29 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
  30 import yaml
  31
  32 # User defined libs
  33 import generate_visState as vis_gen
  34 import generate_uiStateJSON as uiStateJSON_gen
  35 import generate_dashVis as dash_gen
  36 import generate_searchSourceJSON as searchSourceJSON_gen
  37 import data_generate as data_gen
  38
  39
  40 def p(x):
  41     print(json.dumps(x, indent=6, sort_keys=True))
  42
  43
  44 # ELK DB host and port to be passed as ':' separated argument
  45 if len(sys.argv) > 1:
  46     if ":" in sys.argv[1]:
  47         ELK_DB_HOST = sys.argv[1].split(":")[0]
  48         ELK_DB_PORT = sys.argv[1].split(":")[1]
  49 else:
  50     print("Usage: python push_to_elk.py host:port")
  51     print("Unable to publish data to ELK. Exiting.")
  52     sys.exit()
  53
  54 try:
  55     es = Elasticsearch(
  56         hosts=[{"host": ELK_DB_HOST, "port": int(ELK_DB_PORT)}],
  57         scheme="https",
  58         connection_class=RequestsHttpConnection,
  59     )
  60 except Exception as e:
  61     print("Unexpected Error Occurred. Exiting")
  62     print(e)
  63 # print(es.info())
  64
  65
  66 # sys.exit()
  67 # Function to convert JSON object to string.
  68 # Python puts 'true' as 'True' etc. which need handling.
  69
  70
  71 def JSONToString(jobj):
  72     retval = str(jobj)
  73     retval = retval.replace("'", '"')
  74     retval = retval.replace(": ", ":")
  75     retval = retval.replace(", ", ",")
  76     retval = retval.replace("True", "true")
  77     retval = retval.replace("False", "false")
  78     retval = retval.replace("None", "null")
  79     return retval
  80
  81
  82 # Clear .kibana index before pushing visualizations
  83 try:
  84     index = ".kibana"
  85     res = es.indices.delete(index=index)
  86 except Exception as e:
  87     print(e)
  88     # raise e
  89     print("Unable to push data to ElasticSearch")
  90
  91
  92 # Create and push index-pattern to be used by visualizations
  93
  94 TEST_DATA_INDEX = "opendaylight-test"
  95
  96 INDEX_PATTERN_BODY = {
  97     "type": "index-pattern",
  98     "index-pattern": {"timeFieldName": "@timestamp", "title": TEST_DATA_INDEX},
  99 }
 100
 101
 102 KIBANA_CONFIG = {
 103     "config": {
 104         "defaultIndex": "pattern-for-{}".format(TEST_DATA_INDEX),
 105         "timepicker:timeDefaults": '{\n  "from": "now-5y",\n \
 106                                 "to": "now",\n  "mode": "quick"\n}',
 107         "xPackMonitoring:showBanner": False,
 108     },
 109     "type": "config",
 110 }
 111
 112 res = es.index(index=".kibana", doc_type="doc", id="config:6.2.4", body=KIBANA_CONFIG)
 113
 114
 115 try:
 116     index = ".kibana"
 117     ES_ID = "index-pattern:pattern-for-{}".format(TEST_DATA_INDEX)
 118     res = es.index(index=index, doc_type="doc", id=ES_ID, body=INDEX_PATTERN_BODY)
 119     p(json.dumps(INDEX_PATTERN_BODY, indent=4))
 120     print(json.dumps(res, indent=4))
 121 except Exception as e:
 122     print(e)
 123     # raise e
 124     print("Unable to push data to ElasticSearch")
 125
 126 try:
 127     viz_config_path = glob.glob("**/dashboard/viz_config.yaml")[0]
 128 except IndexError:
 129     print("Visualization template file not found!")
 130     sys.exit()
 131
 132 try:
 133     dash_config_path = glob.glob("**/dashboard/dash_config.yaml")[0]
 134 except IndexError:
 135     print("Dashboard configuration file not found!")
 136     sys.exit()
 137
 138 with open(dash_config_path, "r") as f:
 139     dash_config = yaml.safe_load(f)
 140
 141 with open(viz_config_path, "r") as f:
 142     viz_config = yaml.safe_load(f)
 143
 144
 145 # Create and push visualizations
 146 for dashboard_id, dashboard_content in dash_config.items():
 147
 148     for _, i in dash_config[dashboard_id]["viz"].items():
 149         intermediate_format, visState = vis_gen.generate(
 150             i, viz_config[i["viz-template"]]
 151         )
 152
 153         searchSourceJSON = searchSourceJSON_gen.generate(
 154             i, viz_config[i["viz-template"]], intermediate_format["index_pattern"]
 155         )
 156
 157         uiStateJSON = uiStateJSON_gen.generate(i, viz_config[i["viz-template"]])
 158
 159         # p(intermediate_format)
 160         # p(visState)
 161
 162         # Template for visualization template
 163         VIZ_BODY = {
 164             "type": "visualization",
 165             "visualization": {
 166                 "title": None,
 167                 "visState": None,
 168                 "uiStateJSON": "{}",
 169                 "description": None,
 170                 "version": 1,
 171                 "kibanaSavedObjectMeta": {"searchSourceJSON": None},
 172             },
 173         }
 174
 175         VIZ_BODY["visualization"]["title"] = intermediate_format["title"]
 176         VIZ_BODY["visualization"]["visState"] = JSONToString(visState)
 177         VIZ_BODY["visualization"]["uiStateJSON"] = JSONToString(uiStateJSON)
 178         VIZ_BODY["visualization"]["description"] = intermediate_format["desc"]
 179         VIZ_BODY["visualization"]["kibanaSavedObjectMeta"][
 180             "searchSourceJSON"
 181         ] = JSONToString(searchSourceJSON)
 182
 183         p(VIZ_BODY)
 184         # Pushing visualization to Kibana
 185         index = ".kibana"
 186         ES_ID = "visualization:{}".format(i["id"])
 187         res = es.index(index=index, doc_type="doc", id=ES_ID, body=VIZ_BODY)
 188         print(json.dumps(res, indent=4))
 189
 190     # Create and push dashboards
 191
 192     # Template for dashboard body in Kibana
 193     DASH_BODY = {
 194         "type": "dashboard",
 195         "dashboard": {
 196             "title": None,
 197             "description": None,
 198             "panelsJSON": None,
 199             "optionsJSON": '{"darkTheme":false,\
 200                             "hidePanelTitles":false,"useMargins":true}',
 201             "version": 1,
 202             "kibanaSavedObjectMeta": {
 203                 "searchSourceJSON": '{"query":{"language":"lucene", \
 204                                      "query":""}, \
 205                                      "filter":[],"highlightAll" \
 206                                       :true,"version":true}'
 207             },
 208         },
 209     }
 210
 211     DASH_BODY["dashboard"]["title"] = dashboard_content["title"]
 212     DASH_BODY["dashboard"]["description"] = dashboard_content["desc"]
 213     DASH_BODY["dashboard"]["panelsJSON"] = JSONToString(
 214         dash_gen.generate(dashboard_content["viz"])
 215     )
 216
 217     p(DASH_BODY)
 218     # Pushing dashboard to kibana
 219     index = ".kibana"
 220     ES_ID = "dashboard:{}".format(dashboard_content["id"])
 221     res = es.index(index=index, doc_type="doc", id=ES_ID, body=DASH_BODY)
 222     print(json.dumps(res, indent=4))