csit/scripts/push_dashboard.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
   5 ##############################################################################
   6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
   7 #
   8 # All rights reserved. This program and the accompanying materials
   9 # are made available under the terms of the Eclipse Public License v1.0
  10 # which accompanies this distribution, and is available at
  11 # http://www.eclipse.org/legal/epl-v10.html
  12 ##############################################################################
  13
  14 """
  15 This script is used to parse dashboard config files, construct
  16 JSON BODY and push it to ELK DB.
  17
  18 Usage: python push_dashboard.py host:port
  19
  20 """
  21
  22 # stdlib
  23 import json
  24 import os
  25 import sys
  26 import glob
  27
  28 # 3rd party lib
  29 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
  30 import yaml
  31
  32 # User defined libs
  33 import generate_visState as vis_gen
  34 import generate_uiStateJSON as uiStateJSON_gen
  35 import generate_dashVis as dash_gen
  36 import generate_searchSourceJSON as searchSourceJSON_gen
  37 import data_generate as data_gen
  38
  39
  40 def p(x):
  41     print(json.dumps(x, indent=6, sort_keys=True))
  42
  43
  44 # ELK DB host and port to be passed as ':' separated argument
  45 if len(sys.argv) > 1:
  46     if ':' in sys.argv[1]:
  47         ELK_DB_HOST = sys.argv[1].split(':')[0]
  48         ELK_DB_PORT = sys.argv[1].split(':')[1]
  49 else:
  50     print('Usage: python push_to_elk.py host:port')
  51     print('Unable to publish data to ELK. Exiting.')
  52     sys.exit()
  53
  54 try:
  55     es = Elasticsearch(
  56         hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
  57         scheme='https',
  58         connection_class=RequestsHttpConnection
  59     )
  60 except Exception as e:
  61     print('Unexpected Error Occurred. Exiting')
  62     print(e)
  63 # print(es.info())
  64
  65
  66 # sys.exit()
  67 # Function to convert JSON object to string.
  68 # Python puts 'true' as 'True' etc. which need handling.
  69
  70
  71 def JSONToString(jobj):
  72     retval = str(jobj)
  73     retval = retval.replace('\'', '"')
  74     retval = retval.replace(': ', ':')
  75     retval = retval.replace(', ', ',')
  76     retval = retval.replace('True', 'true')
  77     retval = retval.replace('False', 'false')
  78     retval = retval.replace('None', 'null')
  79     return retval
  80
  81
  82 # Clear .kibana index before pushing visualizations
  83 try:
  84     index = '.kibana'
  85     res = es.indices.delete(index=index)
  86 except Exception as e:
  87     print(e)
  88     # raise e
  89     print('Unable to push data to ElasticSearch')
  90
  91
  92 # Create and push index-pattern to be used by visualizations
  93
  94 TEST_DATA_INDEX = 'opendaylight-test'
  95
  96 INDEX_PATTERN_BODY = {
  97     "type": "index-pattern",
  98     "index-pattern": {
  99         "timeFieldName": "@timestamp",
 100         "title": TEST_DATA_INDEX
 101     }
 102 }
 103
 104
 105 KIBANA_CONFIG = {'config': {
 106     'defaultIndex': 'pattern-for-{}'.format(TEST_DATA_INDEX),
 107     'timepicker:timeDefaults': '{\n  "from": "now-5y",\n \
 108                                 "to": "now",\n  "mode": "quick"\n}',
 109     'xPackMonitoring:showBanner': False},
 110     'type': 'config',
 111 }
 112
 113 res = es.index(index='.kibana', doc_type='doc',
 114                id='config:6.2.4', body=KIBANA_CONFIG)
 115
 116
 117 try:
 118     index = '.kibana'
 119     ES_ID = 'index-pattern:pattern-for-{}'.format(
 120         TEST_DATA_INDEX)
 121     res = es.index(index=index, doc_type='doc',
 122                    id=ES_ID, body=INDEX_PATTERN_BODY)
 123     p(json.dumps(INDEX_PATTERN_BODY, indent=4))
 124     print(json.dumps(res, indent=4))
 125 except Exception as e:
 126     print(e)
 127     # raise e
 128     print('Unable to push data to ElasticSearch')
 129
 130 # Create and push visualizations
 131 try:
 132     viz_config_path = glob.glob('**/dashboard/viz_config.yaml')[0]
 133 except IndexError:
 134     print('Visualization template file not found!')
 135     sys.exit()
 136
 137 try:
 138     dash_config_path = glob.glob('**/dashboard/dash_config.yaml')[0]
 139 except IndexError:
 140     print('Dashboard configuration file not found!')
 141     sys.exit()
 142
 143 with open(dash_config_path, 'r') as f:
 144     dash_config = yaml.safe_load(f)
 145
 146 with open(viz_config_path, 'r') as f:
 147     viz_config = yaml.safe_load(f)
 148
 149
 150 for dashboard_id, dashboard_content in dash_config.items():
 151
 152     for _, i in dash_config[dashboard_id]['viz'].items():
 153         intermediate_format, visState = vis_gen.generate(
 154             i, viz_config[i['viz-template']])
 155
 156         searchSourceJSON = searchSourceJSON_gen.generate(
 157             i, viz_config[i['viz-template']],
 158             intermediate_format['index_pattern'])
 159
 160         uiStateJSON = uiStateJSON_gen.generate(
 161             i, viz_config[i['viz-template']])
 162
 163         # p(intermediate_format)
 164         # p(visState)
 165
 166         VIZ_BODY = {
 167             'type': 'visualization',
 168             'visualization': {
 169                 "title": None,
 170                 "visState": None,
 171                 "uiStateJSON": "{}",
 172                 "description": None,
 173                 "version": 1,
 174                 "kibanaSavedObjectMeta": {
 175                     "searchSourceJSON": None
 176                 }
 177             }
 178         }
 179
 180         VIZ_BODY['visualization']['title'] = intermediate_format['title']
 181         VIZ_BODY['visualization']['visState'] = JSONToString(visState)
 182         VIZ_BODY['visualization']['uiStateJSON'] = JSONToString(uiStateJSON)
 183         VIZ_BODY['visualization']['description'] = intermediate_format['desc']
 184         VIZ_BODY['visualization']['kibanaSavedObjectMeta']['searchSourceJSON']\
 185             = JSONToString(
 186             searchSourceJSON)
 187
 188         p(VIZ_BODY)
 189         index = '.kibana'
 190         ES_ID = 'visualization:{}'.format(i['id'])
 191         res = es.index(index=index, doc_type='doc', id=ES_ID, body=VIZ_BODY)
 192         print(json.dumps(res, indent=4))
 193
 194     # Create and push dashboards
 195
 196     DASH_BODY = {
 197         'type': 'dashboard',
 198         'dashboard': {
 199             'title': None,
 200             'description': None,
 201             'panelsJSON': None,
 202             'optionsJSON': '{\"darkTheme\":false,\
 203                             \"hidePanelTitles\":false,\"useMargins\":true}',
 204             'version': 1,
 205             'kibanaSavedObjectMeta': {
 206                 'searchSourceJSON': '{\"query\":{\"language\":\"lucene\", \
 207                                      \"query\":\"\"}, \
 208                                      \"filter\":[],\"highlightAll\" \
 209                                       :true,\"version\":true}'
 210             }
 211         }
 212     }
 213
 214     DASH_BODY['dashboard']['title'] = dashboard_content['title']
 215     DASH_BODY['dashboard']['description'] = dashboard_content['desc']
 216     DASH_BODY['dashboard']['panelsJSON'] = JSONToString(
 217         dash_gen.generate(dashboard_content['viz']))
 218
 219     p(DASH_BODY)
 220
 221     index = '.kibana'
 222     ES_ID = 'dashboard:{}'.format(dashboard_content['id'])
 223     res = es.index(index=index, doc_type='doc', id=ES_ID, body=DASH_BODY)
 224     print(json.dumps(res, indent=4))