csit/scripts/push_to_elk.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
   5 ##############################################################################
   6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
   7 #
   8 # All rights reserved. This program and the accompanying materials
   9 # are made available under the terms of the Eclipse Public License v1.0
  10 # which accompanies this distribution, and is available at
  11 # http://www.eclipse.org/legal/epl-v10.html
  12 ##############################################################################
  13
  14 """
  15 This script is used to parse logs, construct JSON BODY and push
  16 it to ELK DB.
  17
  18 Usage: python construct_json.py host:port
  19
  20 JSON body similar to following is \
  21 constructed from robot files, jenkins environment
  22 and plot files available in workspace available post-build.
  23 {
  24     "project": "opendaylight", <- fix string for ODL project
  25     "subject": "test", <- fix string for ODL test
  26     "test-type": "performance", <- if there are csv files, \
  27                                      otherwise "functional"
  28     "jenkins-silo": "releng" <- from Jenkins $SILO
  29     "test-name": "openflowplugin-csit-1node-periodic \
  30                 -bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
  31     "test-run": 289, <- from Jenkins $BUILD_NUMBER
  32     "start-time": "20170612 16:50:04 GMT-07:00",  <- from robot log
  33     "duration": "00:01:05.942", <- from robot log
  34     "pass-tests": 9, <- from robot log
  35     "fail-tests": 0, <- from robot log
  36     "plots": {
  37         "rate": { <- csv filename
  38             "Config DS": 5816.99726601, <- from csv file
  39             "OVS Switch": 5757.05238918, <- from csv file
  40             "Operational DS": 2654.49139945 <- from csv file
  41         },
  42         "time": { <- csv filename
  43             "Config DS": 17.191, <- from csv file
  44             "OVS Switch": 17.37, <- from csv file
  45             "Operational DS": 37.672 <- from csv file
  46         }
  47     }
  48 }
  49 """
  50
  51 # stdlib
  52 import json
  53 import os
  54 import sys
  55 import glob
  56
  57 # 3rd party lib
  58 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
  59 import yaml
  60 import glob
  61
  62 # User defined libs
  63 import generate_visState as vis_gen
  64 import generate_uiStateJSON as uiStateJSON_gen
  65 import generate_dashVis as dash_gen
  66 import generate_searchSourceJSON as searchSourceJSON_gen
  67 import data_generate as data_gen
  68
  69
  70 def p(x):
  71     print(json.dumps(x, indent=6, sort_keys=True))
  72 # ELK DB host and port to be passed as ':' separated argument
  73
  74
  75 if len(sys.argv) > 1:
  76     if ':' in sys.argv[1]:
  77         ELK_DB_HOST = sys.argv[1].split(':')[0]
  78         ELK_DB_PORT = sys.argv[1].split(':')[1]
  79 else:
  80     print('Usage: python push_to_elk.py host:port')
  81     print('Unable to publish data to ELK. Exiting.')
  82     sys.exit()
  83
  84 # Construct json body
  85
  86 # BODY = {}
  87
  88 try:
  89     es = Elasticsearch(
  90         hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
  91         scheme='https',
  92         connection_class=RequestsHttpConnection
  93     )
  94 except Exception as e:
  95     print('Unexpected Error Occurred. Exiting')
  96     print(e)
  97 # print(es.info())
  98
  99
 100 # get data from the user defined script
 101 BODY = data_gen.generate()
 102
 103 print(json.dumps(BODY, indent=4))
 104
 105 # Try to send request to ELK DB.
 106
 107 try:
 108     index = '{}-{}'.format(BODY[BODY['type']]['project'],
 109                            BODY[BODY['type']]['subject'])
 110     ES_ID = '{}:{}-{}'.format(BODY['type'], BODY[BODY['type']]
 111                               ['test-name'], BODY[BODY['type']]['test-run'])
 112     res = es.index(index=index, doc_type='doc', id=ES_ID, body=BODY)
 113     print(json.dumps(res, indent=4))
 114 except Exception as e:
 115     print(e)
 116     print('Unable to push data to ElasticSearch')
 117
 118
 119 # sys.exit()
 120 # Function to convert JSON object to string.
 121 # Python puts 'true' as 'True' etc. which need handling.
 122
 123
 124 def JSONToString(jobj):
 125     retval = str(jobj)
 126     retval = retval.replace('\'', '"')
 127     retval = retval.replace(': ', ':')
 128     retval = retval.replace(', ', ',')
 129     retval = retval.replace('True', 'true')
 130     retval = retval.replace('False', 'false')
 131     retval = retval.replace('None', 'null')
 132     return retval
 133
 134
 135 # Create and push index-pattern to be used by visualizations
 136
 137 INDEX_PATTERN_BODY = {
 138     "type": "index-pattern",
 139     "index-pattern": {
 140         "timeFieldName": "performance.@timestamp",
 141         "title": '{}-{}'.format(BODY[BODY['type']]['project'],
 142                                 BODY[BODY['type']]['subject'])
 143     }
 144 }
 145
 146
 147 KIBANA_CONFIG = {'config': {
 148     'defaultIndex': 'pattern-for-{}-{}'.format(BODY[BODY['type']]['project'],
 149                                                BODY[BODY['type']]['subject']),
 150     'timepicker:timeDefaults': '{\n  "from": "now-5y",\n \
 151                                 "to": "now",\n  "mode": "quick"\n}',
 152     'xPackMonitoring:showBanner': False},
 153     'type': 'config',
 154 }
 155
 156 res = es.index(index='.kibana', doc_type='doc',
 157                id='config:6.2.4', body=KIBANA_CONFIG)
 158
 159
 160 try:
 161     index = '.kibana'
 162     ES_ID = 'index-pattern:pattern-for-{}-{}'.format(
 163         BODY[BODY['type']]['project'], BODY[BODY['type']]['subject'])
 164     res = es.index(index=index, doc_type='doc',
 165                    id=ES_ID, body=INDEX_PATTERN_BODY)
 166     p(json.dumps(INDEX_PATTERN_BODY, indent=4))
 167     print(json.dumps(res, indent=4))
 168 except Exception as e:
 169     print(e)
 170     # raise e
 171     print('Unable to push data to ElasticSearch')
 172
 173 # Create and push visualizations
 174 try:
 175     viz_config_path = glob.glob('**/dashboard/viz_config.yaml')[0]
 176 except IndexError:
 177     print('Visualization template file not found!')
 178     sys.exit()
 179
 180 try:
 181     dash_config_path = glob.glob('**/dashboard/dash_config.yaml')[0]
 182 except IndexError:
 183     print('Dashboard configuration file not found!')
 184     sys.exit()
 185
 186 with open(dash_config_path, 'r') as f:
 187     dash_config = yaml.safe_load(f)
 188
 189 with open(viz_config_path, 'r') as f:
 190     viz_config = yaml.safe_load(f)
 191
 192
 193 for _, i in dash_config['dashboard']['viz'].items():
 194     intermediate_format, visState = vis_gen.generate(
 195         i, viz_config[i['viz-template']])
 196
 197     searchSourceJSON = searchSourceJSON_gen.generate(
 198         i, viz_config[i['viz-template']], intermediate_format['index_pattern'])
 199
 200     uiStateJSON = uiStateJSON_gen.generate(
 201         i, viz_config[i['viz-template']])
 202
 203     # p(intermediate_format)
 204     # p(visState)
 205
 206     VIZ_BODY = {
 207         'type': 'visualization',
 208         'visualization': {
 209             "title": None,
 210             "visState": None,
 211             "uiStateJSON": "{}",
 212             "description": None,
 213             "version": 1,
 214             "kibanaSavedObjectMeta": {
 215                 "searchSourceJSON": None
 216             }
 217         }
 218     }
 219
 220     VIZ_BODY['visualization']['title'] = intermediate_format['title']
 221     VIZ_BODY['visualization']['visState'] = JSONToString(visState)
 222     VIZ_BODY['visualization']['uiStateJSON'] = JSONToString(uiStateJSON)
 223     VIZ_BODY['visualization']['description'] = intermediate_format['desc']
 224     VIZ_BODY['visualization']['kibanaSavedObjectMeta']['searchSourceJSON']  \
 225         = JSONToString(
 226         searchSourceJSON)
 227
 228     p(VIZ_BODY)
 229     index = '.kibana'
 230     ES_ID = 'visualization:{}'.format(i['id'])
 231     res = es.index(index=index, doc_type='doc', id=ES_ID, body=VIZ_BODY)
 232     print(json.dumps(res, indent=4))
 233
 234
 235 # Create and push dashboards
 236
 237
 238 for _, i in dash_config.items():
 239     DASH_BODY = {
 240         'type': 'dashboard',
 241         'dashboard': {
 242             'title': None,
 243             'description': None,
 244             'panelsJSON': None,
 245             'optionsJSON': '{\"darkTheme\":false,\
 246                             \"hidePanelTitles\":false,\"useMargins\":true}',
 247             'version': 1,
 248             'kibanaSavedObjectMeta': {
 249                 'searchSourceJSON': '{\"query\":{\"language\":\"lucene\", \
 250                                      \"query\":\"\"}, \
 251                                      \"filter\":[],\"highlightAll\" \
 252                                       :true,\"version\":true}'
 253             }
 254         }
 255     }
 256
 257     DASH_BODY['dashboard']['title'] = i['title']
 258     DASH_BODY['dashboard']['description'] = i['desc']
 259     DASH_BODY['dashboard']['panelsJSON'] = JSONToString(
 260         dash_gen.generate(i['viz']))
 261
 262     p(DASH_BODY)
 263
 264     index = '.kibana'
 265     ES_ID = 'dashboard:{}'.format(i['id'])
 266     res = es.index(index=index, doc_type='doc', id=ES_ID, body=DASH_BODY)
 267     print(json.dumps(res, indent=4))