csit/scripts/push_to_elk.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
   5 ##############################################################################
   6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
   7 #
   8 # All rights reserved. This program and the accompanying materials
   9 # are made available under the terms of the Eclipse Public License v1.0
  10 # which accompanies this distribution, and is available at
  11 # http://www.eclipse.org/legal/epl-v10.html
  12 ##############################################################################
  13
  14 """
  15 This script is used to parse logs, construct JSON BODY and push
  16 it to ELK DB.
  17
  18 Usage: python construct_json.py host:port
  19
  20 JSON body similar to following is \
  21 constructed from robot files, jenkins environment
  22 and plot files available in workspace available post-build.
  23 {
  24     "project": "opendaylight", <- fix string for ODL project
  25     "subject": "test", <- fix string for ODL test
  26     "test-type": "performance", <- if there are csv files, \
  27                                      otherwise "functional"
  28     "jenkins-silo": "releng" <- from Jenkins $SILO
  29     "test-name": "openflowplugin-csit-1node-periodic \
  30                 -bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
  31     "test-run": 289, <- from Jenkins $BUILD_NUMBER
  32     "start-time": "20170612 16:50:04 GMT-07:00",  <- from robot log
  33     "duration": "00:01:05.942", <- from robot log
  34     "pass-tests": 9, <- from robot log
  35     "fail-tests": 0, <- from robot log
  36     "plots": {
  37         "rate": { <- csv filename
  38             "Config DS": 5816.99726601, <- from csv file
  39             "OVS Switch": 5757.05238918, <- from csv file
  40             "Operational DS": 2654.49139945 <- from csv file
  41         },
  42         "time": { <- csv filename
  43             "Config DS": 17.191, <- from csv file
  44             "OVS Switch": 17.37, <- from csv file
  45             "Operational DS": 37.672 <- from csv file
  46         }
  47     }
  48 }
  49 """
  50
  51 # stdlib
  52 import json
  53 import os
  54 import sys
  55 import glob
  56
  57 # 3rd party lib
  58 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
  59 import yaml
  60 import glob
  61
  62 # User defined libs
  63 import generate_visState as vis_gen
  64 import generate_uiStateJSON as uiStateJSON_gen
  65 import generate_dashVis as dash_gen
  66 import generate_searchSourceJSON as searchSourceJSON_gen
  67 import data_generate as data_gen
  68
  69
  70 def p(x):
  71     print(json.dumps(x, indent=6, sort_keys=True))
  72 # ELK DB host and port to be passed as ':' separated argument
  73
  74
  75 if len(sys.argv) > 1:
  76     if ':' in sys.argv[1]:
  77         ELK_DB_HOST = sys.argv[1].split(':')[0]
  78         ELK_DB_PORT = sys.argv[1].split(':')[1]
  79 else:
  80     print('Usage: python push_to_elk.py host:port')
  81     print('Unable to publish data to ELK. Exiting.')
  82     sys.exit()
  83
  84 # Construct json body
  85
  86 # BODY = {}
  87
  88 try:
  89     es = Elasticsearch(
  90         hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
  91         scheme='https',
  92         connection_class=RequestsHttpConnection
  93     )
  94 except Exception as e:
  95     print('Unexpected Error Occurred. Exiting')
  96     print(e)
  97 # print(es.info())
  98
  99
 100 # get data from the user defined script
 101 BODY = data_gen.generate()
 102
 103 print(json.dumps(BODY, indent=4))
 104
 105 # Try to send request to ELK DB.
 106
 107 try:
 108     index = '{}-{}'.format(BODY['project'],
 109                            BODY['subject'])
 110     ES_ID = '{}:{}-{}'.format(BODY['test-type'], BODY['test-name'],
 111                               BODY['test-run'])
 112     res = es.index(index=index, doc_type='doc', id=ES_ID, body=BODY)
 113     print(json.dumps(res, indent=4))
 114 except Exception as e:
 115     print(e)
 116     print('Unable to push data to ElasticSearch')
 117
 118
 119 # sys.exit()
 120 # Function to convert JSON object to string.
 121 # Python puts 'true' as 'True' etc. which need handling.
 122
 123
 124 def JSONToString(jobj):
 125     retval = str(jobj)
 126     retval = retval.replace('\'', '"')
 127     retval = retval.replace(': ', ':')
 128     retval = retval.replace(', ', ',')
 129     retval = retval.replace('True', 'true')
 130     retval = retval.replace('False', 'false')
 131     retval = retval.replace('None', 'null')
 132     return retval
 133
 134
 135 # Clear .kibana index before pushing visualizations
 136 try:
 137     index = '.kibana'
 138     res = es.indices.delete(index=index)
 139 except Exception as e:
 140     print(e)
 141     # raise e
 142     print('Unable to push data to ElasticSearch')
 143
 144
 145 # Create and push index-pattern to be used by visualizations
 146
 147 INDEX_PATTERN_BODY = {
 148     "type": "index-pattern",
 149     "index-pattern": {
 150         "timeFieldName": "@timestamp",
 151         "title": '{}-{}'.format(BODY['project'],
 152                                 BODY['subject'])
 153     }
 154 }
 155
 156
 157 KIBANA_CONFIG = {'config': {
 158     'defaultIndex': 'pattern-for-{}-{}'.format(BODY['project'],
 159                                                BODY['subject']),
 160     'timepicker:timeDefaults': '{\n  "from": "now-5y",\n \
 161                                 "to": "now",\n  "mode": "quick"\n}',
 162     'xPackMonitoring:showBanner': False},
 163     'type': 'config',
 164 }
 165
 166 res = es.index(index='.kibana', doc_type='doc',
 167                id='config:6.2.4', body=KIBANA_CONFIG)
 168
 169
 170 try:
 171     index = '.kibana'
 172     ES_ID = 'index-pattern:pattern-for-{}-{}'.format(
 173         BODY['project'], BODY['subject'])
 174     res = es.index(index=index, doc_type='doc',
 175                    id=ES_ID, body=INDEX_PATTERN_BODY)
 176     p(json.dumps(INDEX_PATTERN_BODY, indent=4))
 177     print(json.dumps(res, indent=4))
 178 except Exception as e:
 179     print(e)
 180     # raise e
 181     print('Unable to push data to ElasticSearch')
 182
 183 # Create and push visualizations
 184 try:
 185     viz_config_path = glob.glob('**/dashboard/viz_config.yaml')[0]
 186 except IndexError:
 187     print('Visualization template file not found!')
 188     sys.exit()
 189
 190 try:
 191     dash_config_path = glob.glob('**/dashboard/dash_config.yaml')[0]
 192 except IndexError:
 193     print('Dashboard configuration file not found!')
 194     sys.exit()
 195
 196 with open(dash_config_path, 'r') as f:
 197     dash_config = yaml.safe_load(f)
 198
 199 with open(viz_config_path, 'r') as f:
 200     viz_config = yaml.safe_load(f)
 201
 202
 203 for dashboard_id, dashboard_content in dash_config.items():
 204
 205     for _, i in dash_config[dashboard_id]['viz'].items():
 206         intermediate_format, visState = vis_gen.generate(
 207             i, viz_config[i['viz-template']])
 208
 209         searchSourceJSON = searchSourceJSON_gen.generate(
 210             i, viz_config[i['viz-template']],
 211             intermediate_format['index_pattern'])
 212
 213         uiStateJSON = uiStateJSON_gen.generate(
 214             i, viz_config[i['viz-template']])
 215
 216         # p(intermediate_format)
 217         # p(visState)
 218
 219         VIZ_BODY = {
 220             'type': 'visualization',
 221             'visualization': {
 222                 "title": None,
 223                 "visState": None,
 224                 "uiStateJSON": "{}",
 225                 "description": None,
 226                 "version": 1,
 227                 "kibanaSavedObjectMeta": {
 228                     "searchSourceJSON": None
 229                 }
 230             }
 231         }
 232
 233         VIZ_BODY['visualization']['title'] = intermediate_format['title']
 234         VIZ_BODY['visualization']['visState'] = JSONToString(visState)
 235         VIZ_BODY['visualization']['uiStateJSON'] = JSONToString(uiStateJSON)
 236         VIZ_BODY['visualization']['description'] = intermediate_format['desc']
 237         VIZ_BODY['visualization']['kibanaSavedObjectMeta']['searchSourceJSON']\
 238             = JSONToString(
 239             searchSourceJSON)
 240
 241         p(VIZ_BODY)
 242         index = '.kibana'
 243         ES_ID = 'visualization:{}'.format(i['id'])
 244         res = es.index(index=index, doc_type='doc', id=ES_ID, body=VIZ_BODY)
 245         print(json.dumps(res, indent=4))
 246
 247     # Create and push dashboards
 248
 249     DASH_BODY = {
 250         'type': 'dashboard',
 251         'dashboard': {
 252             'title': None,
 253             'description': None,
 254             'panelsJSON': None,
 255             'optionsJSON': '{\"darkTheme\":false,\
 256                             \"hidePanelTitles\":false,\"useMargins\":true}',
 257             'version': 1,
 258             'kibanaSavedObjectMeta': {
 259                 'searchSourceJSON': '{\"query\":{\"language\":\"lucene\", \
 260                                      \"query\":\"\"}, \
 261                                      \"filter\":[],\"highlightAll\" \
 262                                       :true,\"version\":true}'
 263             }
 264         }
 265     }
 266
 267     DASH_BODY['dashboard']['title'] = dashboard_content['title']
 268     DASH_BODY['dashboard']['description'] = dashboard_content['desc']
 269     DASH_BODY['dashboard']['panelsJSON'] = JSONToString(
 270         dash_gen.generate(dashboard_content['viz']))
 271
 272     p(DASH_BODY)
 273
 274     index = '.kibana'
 275     ES_ID = 'dashboard:{}'.format(dashboard_content['id'])
 276     res = es.index(index=index, doc_type='doc', id=ES_ID, body=DASH_BODY)
 277     print(json.dumps(res, indent=4))