2 # -*- coding: utf-8 -*-
4 # @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
5 ##############################################################################
6 # Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
8 # All rights reserved. This program and the accompanying materials
9 # are made available under the terms of the Eclipse Public License v1.0
10 # which accompanies this distribution, and is available at
11 # http://www.eclipse.org/legal/epl-v10.html
12 ##############################################################################
15 This script is used to parse test logs, construct JSON BODY and push
18 Usage: python push_test_data.py host:port
20 JSON body similar to following is \
21 constructed from robot files, jenkins environment
22 and plot files available in workspace available post-build.
24 "project": "opendaylight", <- fix string for ODL project
25 "subject": "test", <- fix string for ODL test
26 "test-type": "performance", <- if there are csv files, \
27 otherwise "functional"
28 "jenkins-silo": "releng" <- from Jenkins $SILO
29 "test-name": "openflowplugin-csit-1node-periodic \
30 -bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
31 "test-run": 289, <- from Jenkins $BUILD_NUMBER
32 "start-time": "20170612 16:50:04 GMT-07:00", <- from robot log
33 "duration": "00:01:05.942", <- from robot log
34 "pass-tests": 9, <- from robot log
35 "fail-tests": 0, <- from robot log
37 "rate": { <- csv filename
38 "Config DS": 5816.99726601, <- from csv file
39 "OVS Switch": 5757.05238918, <- from csv file
40 "Operational DS": 2654.49139945 <- from csv file
42 "time": { <- csv filename
43 "Config DS": 17.191, <- from csv file
44 "OVS Switch": 17.37, <- from csv file
45 "Operational DS": 37.672 <- from csv file
56 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
59 import data_generate as data_gen
63 print(json.dumps(x, indent=6, sort_keys=True))
66 # ELK DB host and port to be passed as ':' separated argument
70 if ":" in sys.argv[1]:
71 ELK_DB_HOST = sys.argv[1].split(":")[0]
72 ELK_DB_PORT = sys.argv[1].split(":")[1]
74 print("Usage: python push_to_elk.py host:port")
75 print("Unable to publish data to ELK. Exiting.")
84 hosts=[{"host": ELK_DB_HOST, "port": int(ELK_DB_PORT)}],
86 connection_class=RequestsHttpConnection,
88 except Exception as e:
89 print("Unexpected Error Occurred. Exiting")
94 # get data from the user defined script
95 BODY = data_gen.generate()
97 print(json.dumps(BODY, indent=4))
99 # Skip ELK update if it comes from sandbox.
100 if BODY["jenkins-silo"] == "sandbox":
101 print("silo is sandbox, ELK update is skipped")
104 # Try to send request to ELK DB.
106 index = "{}-{}".format(BODY["project"], BODY["subject"])
107 ES_ID = "{}:{}-{}".format(BODY["test-type"], BODY["test-name"], BODY["test-run"])
108 res = es.index(index=index, doc_type="doc", id=ES_ID, body=BODY)
109 print(json.dumps(res, indent=4))
110 except Exception as e:
112 print("Unable to push data to ElasticSearch")