3 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
4 ##############################################################################
5 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
7 # All rights reserved. This program and the accompanying materials
8 # are made available under the terms of the Eclipse Public License v1.0
9 # which accompanies this distribution, and is available at
10 # http://www.eclipse.org/legal/epl-v10.html
11 ##############################################################################
14 This script is used to parse logs, construct JSON BODY and push
17 Usage: python construct_json.py host:port
19 JSON body similar to following is constructed from robot files, jenkins environment
20 and plot files available in workspace available post-build.
22 "project": "opendaylight", <- fix string for ODL project
23 "subject": "test", <- fix string for ODL test
24 "test-type": "performance", <- if there are csv files, otherwise "functional"
25 "jenkins-silo": "releng" <- from Jenkins $SILO
26 "test-name": "openflowplugin-csit-1node-periodic-bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
27 "test-run": 289, <- from Jenkins $BUILD_NUMBER
28 "start-time": "20170612 16:50:04 GMT-07:00", <- from robot log
29 "duration": "00:01:05.942", <- from robot log
30 "pass-tests": 9, <- from robot log
31 "fail-tests": 0, <- from robot log
33 "rate": { <- csv filename
34 "Config DS": 5816.99726601, <- from csv file
35 "OVS Switch": 5757.05238918, <- from csv file
36 "Operational DS": 2654.49139945 <- from csv file
38 "time": { <- csv filename
39 "Config DS": 17.191, <- from csv file
40 "OVS Switch": 17.37, <- from csv file
41 "Operational DS": 37.672 <- from csv file
48 from datetime import datetime
55 import xml.etree.ElementTree as ET
58 # ELK DB host and port to be passed as ':' separated argument
60 if ':' in sys.argv[1]:
61 ELK_DB_HOST = sys.argv[1].split(':')[0]
62 ELK_DB_PORT = sys.argv[1].split(':')[1]
64 print("Usage: python push_to_elk.py host:port")
65 print("Unable to publish data to ELK. Exiting.")
72 formatted_ts = datetime.fromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
73 BODY['@timestamp'] = formatted_ts
74 # Plots are obtained from csv files (present in archives directory in $WORKSPACE).
75 csv_files = glob.glob('archives/*.csv')
76 BODY['project'] = 'opendaylight'
77 BODY['subject'] = 'test'
79 # If there are no csv files, then it is a functional test.
80 # Parse csv files and fill perfomance parameter values
81 if (len(csv_files) == 0):
82 BODY['test-type'] = 'functional'
84 BODY['test-type'] = 'performance'
87 key = f.split('/')[-1][:-4]
88 BODY['plots'][key] = {}
89 lines = open(f).readlines()
90 props = lines[0].strip().split(',')
91 vals = lines[1].strip().split(',')
92 BODY['plots'][key][props[0]] = float(vals[0])
93 BODY['plots'][key][props[1]] = float(vals[1])
94 BODY['plots'][key][props[2]] = float(vals[2])
96 # Fill the required parameters whose values are obtained from environment.
97 BODY['jenkins-silo'] = os.environ['SILO']
98 BODY['test-name'] = os.environ['JOB_NAME']
99 BODY['test-run'] = os.environ['BUILD_NUMBER']
101 # Parsing robot log for statistics on no of start-time, pass/fail tests and duration.
102 robot_log = os.environ['WORKSPACE'] + '/output.xml'
103 tree = ET.parse(robot_log)
104 BODY['id'] = '{}-{}'.format(os.environ['JOB_NAME'], os.environ['BUILD_NUMBER'])
105 BODY['start-time'] = tree.getroot().attrib['generated']
106 BODY['pass-tests'] = tree.getroot().find('statistics')[0][1].get('pass')
107 BODY['fail-tests'] = tree.getroot().find('statistics')[0][1].get('fail')
108 endtime = tree.getroot().find('suite').find('status').get('endtime')
109 starttime = tree.getroot().find('suite').find('status').get('starttime')
110 elap_time = datetime.strptime(endtime, '%Y%m%d %H:%M:%S.%f') - datetime.strptime(starttime, '%Y%m%d %H:%M:%S.%f')
111 BODY['duration'] = str(elap_time)
113 # Parse JSON BODY to construct PUT_URL
114 PUT_URL_INDEX = '/{}-{}'.format(BODY['project'], BODY['subject'])
115 PUT_URL_TYPE = '/{}'.format(BODY['test-type'])
116 PUT_URL_ID = '/{}-{}'.format(BODY['test-name'], BODY['test-run'])
117 PUT_URL = 'https://{}:{}{}{}{}'.format(ELK_DB_HOST, ELK_DB_PORT, PUT_URL_INDEX, PUT_URL_TYPE, PUT_URL_ID)
120 print(json.dumps(BODY, indent=4))
122 # Try to send request to ELK DB.
124 r = requests.put(PUT_URL, json=BODY)
128 print('Unable to send PUT request')