2 # -*- coding: utf-8 -*-
4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
5 ##############################################################################
6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
8 # All rights reserved. This program and the accompanying materials
9 # are made available under the terms of the Eclipse Public License v1.0
10 # which accompanies this distribution, and is available at
11 # http://www.eclipse.org/legal/epl-v10.html
12 ##############################################################################
15 This script is used to parse logs, construct JSON BODY and push
18 Usage: python construct_json.py host:port
20 JSON body similar to following is constructed from robot files, jenkins environment
21 and plot files available in workspace available post-build.
23 "project": "opendaylight", <- fix string for ODL project
24 "subject": "test", <- fix string for ODL test
25 "test-type": "performance", <- if there are csv files, otherwise "functional"
26 "jenkins-silo": "releng" <- from Jenkins $SILO
27 "test-name": "openflowplugin-csit-1node-periodic-bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
28 "test-run": 289, <- from Jenkins $BUILD_NUMBER
29 "start-time": "20170612 16:50:04 GMT-07:00", <- from robot log
30 "duration": "00:01:05.942", <- from robot log
31 "pass-tests": 9, <- from robot log
32 "fail-tests": 0, <- from robot log
34 "rate": { <- csv filename
35 "Config DS": 5816.99726601, <- from csv file
36 "OVS Switch": 5757.05238918, <- from csv file
37 "Operational DS": 2654.49139945 <- from csv file
39 "time": { <- csv filename
40 "Config DS": 17.191, <- from csv file
41 "OVS Switch": 17.37, <- from csv file
42 "Operational DS": 37.672 <- from csv file
49 from datetime import datetime
55 import xml.etree.ElementTree as ET
58 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
61 # ELK DB host and port to be passed as ':' separated argument
64 if ':' in sys.argv[1]:
65 ELK_DB_HOST = sys.argv[1].split(':')[0]
66 ELK_DB_PORT = sys.argv[1].split(':')[1]
68 print('Usage: python push_to_elk.py host:port')
69 print('Unable to publish data to ELK. Exiting.')
78 hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
79 connection_class=RequestsHttpConnection
81 except Exception as e:
82 print('Unexpected Error Occurred. Exiting')
89 datetime.fromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
90 BODY['@timestamp'] = formatted_ts
92 # Plots are obtained from csv files (present in archives directory in $WORKSPACE).
94 csv_files = glob.glob('archives/*.csv')
95 BODY['project'] = 'opendaylight'
96 BODY['subject'] = 'test'
98 # If there are no csv files, then it is a functional test.
99 # Parse csv files and fill perfomance parameter values
101 if len(csv_files) == 0:
102 BODY['test-type'] = 'functional'
104 BODY['test-type'] = 'performance'
107 key = (f.split('/')[-1])[:-4]
108 BODY['plots'][key] = {}
109 with open(f) as file:
110 lines = file.readlines()
111 props = lines[0].strip().split(',')
112 vals = lines[1].strip().split(',')
113 BODY['plots'][key][props[0]] = float(vals[0])
114 BODY['plots'][key][props[1]] = float(vals[1])
115 BODY['plots'][key][props[2]] = float(vals[2])
117 # Fill the required parameters whose values are obtained from environment.
119 BODY['jenkins-silo'] = os.environ['SILO']
120 BODY['test-name'] = os.environ['JOB_NAME']
121 BODY['test-run'] = os.environ['BUILD_NUMBER']
123 # Parsing robot log for statistics on no of start-time, pass/fail tests and duration.
125 robot_log = os.environ['WORKSPACE'] + '/output.xml'
126 tree = ET.parse(robot_log)
127 BODY['id'] = '{}-{}'.format(os.environ['JOB_NAME'],
128 os.environ['BUILD_NUMBER'])
129 BODY['start-time'] = tree.getroot().attrib['generated']
130 BODY['pass-tests'] = tree.getroot().find('statistics')[0][1].get('pass')
131 BODY['fail-tests'] = tree.getroot().find('statistics')[0][1].get('fail')
132 endtime = tree.getroot().find('suite').find('status').get('endtime')
133 starttime = tree.getroot().find('suite').find('status').get('starttime')
134 elap_time = datetime.strptime(endtime, '%Y%m%d %H:%M:%S.%f') \
135 - datetime.strptime(starttime, '%Y%m%d %H:%M:%S.%f')
136 BODY['duration'] = str(elap_time)
138 print(json.dumps(BODY, indent=4))
140 # Try to send request to ELK DB.
143 index = '{}-{}'.format(BODY['project'], BODY['subject'])
144 ES_ID = '{}-{}'.format(BODY['test-name'], BODY['test-run'])
145 res = es.index(index=index, doc_type=BODY['test-type'], id=ES_ID, body=BODY)
146 print(json.dumps(res, indent=4))
147 except Exception as e:
149 print('Unable to push data to ElasticSearch')
151 # Function to convert JSON object to string.
152 # Python puts 'true' as 'True' etc. which need handling.
155 def JSONToString(jobj):
157 retval = retval.replace('\'', '"')
158 retval = retval.replace(': ', ':')
159 retval = retval.replace(', ', ',')
160 retval = retval.replace('True', 'true')
161 retval = retval.replace('False', 'false')
162 retval = retval.replace('None', 'null')
166 # This function takes
167 # testname (string, eg: 'openflowplugin-csit-1node-periodic-bulkomatic-perf-daily-only-carbon'),
168 # fieldlist (list of fields, eg: ['pass-tests', 'failed-tests']),
169 # plotkey (string, eg: 'rate')
170 # as parameters and constructs a visualization object in JSON format
172 def getVisualization(testname, fieldlist, plotkey=''):
174 vis['title'] = testname
176 vis['title'] += '-' + plotkey
177 vis['description'] = 'visualization of ' + plotkey \
178 + ' trends for testplan ' + testname
180 vis['kibanaSavedObjectMeta'] = {'searchSourceJSON': ''}
182 'index': 'opendaylight-test',
185 'analyze_wildcard': True,
191 'index': 'opendaylight-test',
213 vis['kibanaSavedObjectMeta']['searchSourceJSON'] = \
214 JSONToString(searchSourceJSON)
215 vis['uiStateJSON'] = '{"vis":{"legendOpen":true, "colors":{"pass-tests":"#7EB26D","failed-tests":"#E24D42"}}}'
217 'title': vis['title'],
221 'addTimeMarker': False,
225 'categoryLines': False,
230 'legendPosition': 'right',
233 'id': 'CategoryAxis-1',
234 'labels': {'show': True, 'truncate': 100},
235 'position': 'bottom',
236 'scale': {'type': 'linear'},
239 'title': {'text': 'Test run number'},
250 'name': 'LeftAxis-1',
252 'scale': {'mode': 'normal', 'type': 'linear'},
255 'title': {'text': ''},
267 'extended_bounds': {},
268 'customLabel': 'Test run number',
273 if plotkey != '': # Performance plot
274 visState['type'] = 'line'
275 for field in fieldlist:
280 'drawLinesBetweenPoints': True,
282 'interpolate': 'linear',
285 'id': str(len(visState['params']['seriesParams']) + 1) + '-' + vis['title'],
286 'label': field.split('.')[-1]
288 'valueAxis': 'ValueAxis-1',
290 if plotkey != '': # Performance plot
291 seriesParam['type'] = 'line'
293 'id': str(len(visState['params']['seriesParams']) + 1) + '-' + vis['title'],
299 'customLabel': field.split('.')[-1]
303 visState['params']['seriesParams'].append(seriesParam)
304 visState['aggs'].append(agg)
306 vis['visState'] = JSONToString(visState)
311 if BODY['test-type'] == 'performance':
313 # Create visualizations for performance tests
314 # One visualization for one plot
316 for key in BODY['plots']:
318 for subkey in BODY['plots'][key]:
319 fieldlist.append('plots.' + key + '.' + subkey)
320 vis = getVisualization(BODY['test-name'], fieldlist, key)
321 vis_ids.append(BODY['test-name'] + '-' + key)
322 print(json.dumps(vis, indent=4))
324 ES_ID = '{}-{}'.format(BODY['test-name'], key)
325 res = es.index(index='.kibana', doc_type='visualization', id=ES_ID, body=BODY)
326 print(json.dumps(res, indent=4))
327 except Exception as e:
329 print('Unable to push visualization to Kibana')
332 vis = getVisualization(BODY['test-name'],
333 ['pass-tests', 'failed-tests'])
334 vis_ids.append(BODY['test-name'])
336 print(json.dumps(vis, indent=4))
338 ES_ID = BODY['test-name']
339 res = es.index(index='.kibana', doc_type='visualization', id=ES_ID, body=vis)
340 print(json.dumps(res, indent=4))
341 except Exception as e:
343 print('Unable to push dashboard to Kibana')
345 # Create dashboard and add above created visualizations to it
347 DASHBOARD_NAME = BODY['test-name'].split('-')[0]
349 dashboard['title'] = DASHBOARD_NAME
350 dashboard['description'] = 'Dashboard for visualizing ' \
352 dashboard['uiStateJSON'] = '{}'
353 dashboard['optionsJSON'] = '{"darkTheme":false}'
354 dashboard['version'] = 1
355 dashboard['timeRestore'] = False
356 dashboard['kibanaSavedObjectMeta'] = {
357 'searchSourceJSON': '{"filter":[{"query":{"query_string":{"query":"*","analyze_wildcard":true}}}],'
358 '"highlightAll":true,"version":true}'
361 # Check if visualizations already present in dashboard. If present, don't add, else, add at end
363 ES_ID = DASHBOARD_NAME
365 res = es.get(index='.kibana', doc_type='dashboard', id=ES_ID)
366 print(json.dumps(res, indent=4))
367 # No exeception occured means dashboard found
368 dashboard_found = True
369 except exceptions.NotFoundError as e:
370 print('No visualizations found')
371 dashboard_found = False
372 except Exception as e:
374 print('Error Occurred')
376 vis_ids_present = set()
381 panelsJSON = yaml.safe_load(res['_source']['panelsJSON'])
382 for vis in panelsJSON:
383 vis_ids_present.add(vis['id'])
387 xpos = (len(vis_ids_present) % 2) * 6 + 1
388 ypos = (len(vis_ids_present) / 2) * 3 + 1
389 for (i, vis_id) in enumerate(vis_ids):
390 if (not dashboard_found or vis_id not in vis_ids_present):
394 'panelIndex': len(vis_ids_present) + i,
395 'type': 'visualization',
404 panelsJSON.append(panelJSON)
406 print('visualization ' + vis_id + ' already present in dashboard')
408 dashboard['panelsJSON'] = JSONToString(panelsJSON)
410 print(json.dumps(dashboard, indent=4))
413 ES_ID = DASHBOARD_NAME
414 res = es.index(index='.kibana', doc_type='dashboard', id=ES_ID, body=dashboard)
415 print(json.dumps(res, indent=4))
416 except exceptions.TransportError as et:
418 print('Elasticsearch returned an error')
419 except Exception as e:
421 print('Unexpected error occurred. Unable to push dashboard to Kibana')