2 # -*- coding: utf-8 -*-
4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
5 ##############################################################################
6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
8 # All rights reserved. This program and the accompanying materials
9 # are made available under the terms of the Eclipse Public License v1.0
10 # which accompanies this distribution, and is available at
11 # http://www.eclipse.org/legal/epl-v10.html
12 ##############################################################################
15 This script is used to parse logs, construct JSON BODY and push
18 Usage: python construct_json.py host:port
20 JSON body similar to following is \
21 constructed from robot files, jenkins environment
22 and plot files available in workspace available post-build.
24 "project": "opendaylight", <- fix string for ODL project
25 "subject": "test", <- fix string for ODL test
26 "test-type": "performance", <- if there are csv files, \
27 otherwise "functional"
28 "jenkins-silo": "releng" <- from Jenkins $SILO
29 "test-name": "openflowplugin-csit-1node-periodic \
30 -bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
31 "test-run": 289, <- from Jenkins $BUILD_NUMBER
32 "start-time": "20170612 16:50:04 GMT-07:00", <- from robot log
33 "duration": "00:01:05.942", <- from robot log
34 "pass-tests": 9, <- from robot log
35 "fail-tests": 0, <- from robot log
37 "rate": { <- csv filename
38 "Config DS": 5816.99726601, <- from csv file
39 "OVS Switch": 5757.05238918, <- from csv file
40 "Operational DS": 2654.49139945 <- from csv file
42 "time": { <- csv filename
43 "Config DS": 17.191, <- from csv file
44 "OVS Switch": 17.37, <- from csv file
45 "Operational DS": 37.672 <- from csv file
58 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
63 import generate_visState as vis_gen
64 import generate_uiStateJSON as uiStateJSON_gen
65 import generate_dashVis as dash_gen
66 import generate_searchSourceJSON as searchSourceJSON_gen
67 import data_generate as data_gen
71 print(json.dumps(x, indent=6, sort_keys=True))
72 # ELK DB host and port to be passed as ':' separated argument
76 if ':' in sys.argv[1]:
77 ELK_DB_HOST = sys.argv[1].split(':')[0]
78 ELK_DB_PORT = sys.argv[1].split(':')[1]
80 print('Usage: python push_to_elk.py host:port')
81 print('Unable to publish data to ELK. Exiting.')
90 hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
92 connection_class=RequestsHttpConnection
94 except Exception as e:
95 print('Unexpected Error Occurred. Exiting')
100 # get data from the user defined script
101 BODY = data_gen.generate()
103 print(json.dumps(BODY, indent=4))
105 # Try to send request to ELK DB.
108 index = '{}-{}'.format(BODY['project'],
110 ES_ID = '{}:{}-{}'.format(BODY['test-type'], BODY['test-name'],
112 res = es.index(index=index, doc_type='doc', id=ES_ID, body=BODY)
113 print(json.dumps(res, indent=4))
114 except Exception as e:
116 print('Unable to push data to ElasticSearch')
120 # Function to convert JSON object to string.
121 # Python puts 'true' as 'True' etc. which need handling.
124 def JSONToString(jobj):
126 retval = retval.replace('\'', '"')
127 retval = retval.replace(': ', ':')
128 retval = retval.replace(', ', ',')
129 retval = retval.replace('True', 'true')
130 retval = retval.replace('False', 'false')
131 retval = retval.replace('None', 'null')
135 # Create and push index-pattern to be used by visualizations
137 INDEX_PATTERN_BODY = {
138 "type": "index-pattern",
140 "timeFieldName": "@timestamp",
141 "title": '{}-{}'.format(BODY['project'],
147 KIBANA_CONFIG = {'config': {
148 'defaultIndex': 'pattern-for-{}-{}'.format(BODY['project'],
150 'timepicker:timeDefaults': '{\n "from": "now-5y",\n \
151 "to": "now",\n "mode": "quick"\n}',
152 'xPackMonitoring:showBanner': False},
156 res = es.index(index='.kibana', doc_type='doc',
157 id='config:6.2.4', body=KIBANA_CONFIG)
162 ES_ID = 'index-pattern:pattern-for-{}-{}'.format(
163 BODY['project'], BODY['subject'])
164 res = es.index(index=index, doc_type='doc',
165 id=ES_ID, body=INDEX_PATTERN_BODY)
166 p(json.dumps(INDEX_PATTERN_BODY, indent=4))
167 print(json.dumps(res, indent=4))
168 except Exception as e:
171 print('Unable to push data to ElasticSearch')
173 # Create and push visualizations
175 viz_config_path = glob.glob('**/dashboard/viz_config.yaml')[0]
177 print('Visualization template file not found!')
181 dash_config_path = glob.glob('**/dashboard/dash_config.yaml')[0]
183 print('Dashboard configuration file not found!')
186 with open(dash_config_path, 'r') as f:
187 dash_config = yaml.safe_load(f)
189 with open(viz_config_path, 'r') as f:
190 viz_config = yaml.safe_load(f)
193 for dashboard_id, dashboard_content in dash_config.items():
195 for _, i in dash_config[dashboard_id]['viz'].items():
196 intermediate_format, visState = vis_gen.generate(
197 i, viz_config[i['viz-template']])
199 searchSourceJSON = searchSourceJSON_gen.generate(
200 i, viz_config[i['viz-template']],
201 intermediate_format['index_pattern'])
203 uiStateJSON = uiStateJSON_gen.generate(
204 i, viz_config[i['viz-template']])
206 # p(intermediate_format)
210 'type': 'visualization',
217 "kibanaSavedObjectMeta": {
218 "searchSourceJSON": None
223 VIZ_BODY['visualization']['title'] = intermediate_format['title']
224 VIZ_BODY['visualization']['visState'] = JSONToString(visState)
225 VIZ_BODY['visualization']['uiStateJSON'] = JSONToString(uiStateJSON)
226 VIZ_BODY['visualization']['description'] = intermediate_format['desc']
227 VIZ_BODY['visualization']['kibanaSavedObjectMeta']['searchSourceJSON']\
233 ES_ID = 'visualization:{}'.format(i['id'])
234 res = es.index(index=index, doc_type='doc', id=ES_ID, body=VIZ_BODY)
235 print(json.dumps(res, indent=4))
237 # Create and push dashboards
245 'optionsJSON': '{\"darkTheme\":false,\
246 \"hidePanelTitles\":false,\"useMargins\":true}',
248 'kibanaSavedObjectMeta': {
249 'searchSourceJSON': '{\"query\":{\"language\":\"lucene\", \
251 \"filter\":[],\"highlightAll\" \
252 :true,\"version\":true}'
257 DASH_BODY['dashboard']['title'] = dashboard_content['title']
258 DASH_BODY['dashboard']['description'] = dashboard_content['desc']
259 DASH_BODY['dashboard']['panelsJSON'] = JSONToString(
260 dash_gen.generate(dashboard_content['viz']))
265 ES_ID = 'dashboard:{}'.format(dashboard_content['id'])
266 res = es.index(index=index, doc_type='doc', id=ES_ID, body=DASH_BODY)
267 print(json.dumps(res, indent=4))