Also use configured series value to sort series.
Change-Id: Idf09eb04c905e0f75c07fbf1123170c684a9df7a
Signed-off-by: Luis Gomez <ecelgp@gmail.com>
# Place the commands in run order:
-python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_test_data.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
# Place the commands in run order:
-python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_test_data.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
# Place the commands in run order:
-python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_test_data.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
# Place the commands in run order:
-python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_test_data.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
# Place the commands in run order:
-python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_test_data.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
# Place the commands in run order:
-python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_test_data.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
agg = aggs()
- temp['aggs'] = [dc(agg.create(field=i['field'],
- custom_label=i['custom_label'],
- schema=i['schema']))
- for _, i in
- config['aggs'].items()]
+ temp['aggs'] = \
+ [dc(agg.create(id=i,
+ field=config['aggs'][i]['field'],
+ custom_label=config['aggs'][i]['custom_label'],
+ schema=config['aggs'][i]['schema']))
+ for i in range(1, len(config['aggs']) + 1)]
temp['params']['seriesParams'] = [seriesParams(i['data_type'],
i['mode'],
}
self.counter = 0
- def create(self, field, custom_label, schema):
- self.counter += 1
+ def create(self, id, field, custom_label, schema):
temp = dc(self.content)
- temp['id'] = str(self.counter)
+ temp['id'] = id
temp['params']['field'] = field
temp['params']['customLabel'] = custom_label
temp['schema'] = schema
format['value_axes'].update(temp)
value_axes_counter += 1
- seriesParams_counter = 1
seriesParams_fields = ['value_axis',
'data_type', 'mode', 'label', 'agg_id']
try:
for m in viz_config['seriesParams']:
if m != 'default':
temp = dc(seriesParams_format)
- temp[str(seriesParams_counter)] = temp['index']
+ temp[m] = temp['index']
for i in seriesParams_fields:
try:
- temp[str(seriesParams_counter)
- ][i] = viz_config['seriesParams'][m][i]
+ temp[m][i] = viz_config['seriesParams'][m][i]
except KeyError:
pass
format['seriesParams'].update(temp)
- seriesParams_counter += 1
except KeyError:
pass
for m in viz_config['aggs']:
if m != 'default':
temp = dc(aggs_format)
- temp[str(agg_counter)] = temp['index']
+ temp[m] = temp['index']
for i in ['field', 'custom_label', 'schema']:
try:
- temp[str(agg_counter)][i] = viz_config['aggs'][m][i]
+ temp[m][i] = viz_config['aggs'][m][i]
except KeyError:
pass
format['aggs'].update(temp)
- agg_counter += 1
except KeyError:
pass
config['series'][key]['not_in_seriesParams']
except KeyError:
seriesParams_temp = dc(seriesParams_format)
- seriesParams_temp[str(
- seriesParams_counter)] = seriesParams_temp['index']
+ seriesParams_temp[key] = seriesParams_temp['index']
for index in ['value_axis', 'data_type', 'mode', 'label']:
try:
- seriesParams_temp[str(
- seriesParams_counter)][index] = \
+ seriesParams_temp[key][index] = \
config['series'][key][index]
except KeyError as e:
pass
- seriesParams_temp[str(
- seriesParams_counter)]['agg_id'] = agg_counter
+ seriesParams_temp[key]['agg_id'] = key
format['seriesParams'].update(seriesParams_temp)
- seriesParams_counter += 1
finally:
agg_temp = dc(aggs_format)
- agg_temp[str(agg_counter)] = agg_temp['index']
+ agg_temp[key] = agg_temp['index']
for index in ['field', 'schema']:
try:
- agg_temp[str(agg_counter)
- ][index] = config['series'][key][index]
+ agg_temp[key][index] = config['series'][key][index]
except KeyError as e:
pass
- agg_temp[str(
- agg_counter)]['custom_label'] = \
+ agg_temp[key]['custom_label'] = \
config['series'][key]['label']
format['aggs'].update(agg_temp)
- agg_counter += 1
except KeyError as e:
print("required fields are empty!")
##############################################################################
"""
-This script is used to parse logs, construct JSON BODY and push
-it to ELK DB.
-
-Usage: python construct_json.py host:port
-
-JSON body similar to following is \
-constructed from robot files, jenkins environment
-and plot files available in workspace available post-build.
-{
- "project": "opendaylight", <- fix string for ODL project
- "subject": "test", <- fix string for ODL test
- "test-type": "performance", <- if there are csv files, \
- otherwise "functional"
- "jenkins-silo": "releng" <- from Jenkins $SILO
- "test-name": "openflowplugin-csit-1node-periodic \
- -bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
- "test-run": 289, <- from Jenkins $BUILD_NUMBER
- "start-time": "20170612 16:50:04 GMT-07:00", <- from robot log
- "duration": "00:01:05.942", <- from robot log
- "pass-tests": 9, <- from robot log
- "fail-tests": 0, <- from robot log
- "plots": {
- "rate": { <- csv filename
- "Config DS": 5816.99726601, <- from csv file
- "OVS Switch": 5757.05238918, <- from csv file
- "Operational DS": 2654.49139945 <- from csv file
- },
- "time": { <- csv filename
- "Config DS": 17.191, <- from csv file
- "OVS Switch": 17.37, <- from csv file
- "Operational DS": 37.672 <- from csv file
- }
- }
-}
+This script is used to parse dashboard config files, construct
+JSON BODY and push it to ELK DB.
+
+Usage: python push_dashboard.py host:port
+
"""
# stdlib
# 3rd party lib
from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
import yaml
-import glob
# User defined libs
import generate_visState as vis_gen
def p(x):
print(json.dumps(x, indent=6, sort_keys=True))
-# ELK DB host and port to be passed as ':' separated argument
+# ELK DB host and port to be passed as ':' separated argument
if len(sys.argv) > 1:
if ':' in sys.argv[1]:
ELK_DB_HOST = sys.argv[1].split(':')[0]
print('Unable to publish data to ELK. Exiting.')
sys.exit()
-# Construct json body
-
-# BODY = {}
-
try:
es = Elasticsearch(
hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
# print(es.info())
-# get data from the user defined script
-BODY = data_gen.generate()
-
-print(json.dumps(BODY, indent=4))
-
-# Try to send request to ELK DB.
-
-try:
- index = '{}-{}'.format(BODY['project'],
- BODY['subject'])
- ES_ID = '{}:{}-{}'.format(BODY['test-type'], BODY['test-name'],
- BODY['test-run'])
- res = es.index(index=index, doc_type='doc', id=ES_ID, body=BODY)
- print(json.dumps(res, indent=4))
-except Exception as e:
- print(e)
- print('Unable to push data to ElasticSearch')
-
-
# sys.exit()
# Function to convert JSON object to string.
# Python puts 'true' as 'True' etc. which need handling.
# Create and push index-pattern to be used by visualizations
+TEST_DATA_INDEX = 'opendaylight-test'
+
INDEX_PATTERN_BODY = {
"type": "index-pattern",
"index-pattern": {
"timeFieldName": "@timestamp",
- "title": '{}-{}'.format(BODY['project'],
- BODY['subject'])
+ "title": TEST_DATA_INDEX
}
}
KIBANA_CONFIG = {'config': {
- 'defaultIndex': 'pattern-for-{}-{}'.format(BODY['project'],
- BODY['subject']),
+ 'defaultIndex': 'pattern-for-{}'.format(TEST_DATA_INDEX),
'timepicker:timeDefaults': '{\n "from": "now-5y",\n \
"to": "now",\n "mode": "quick"\n}',
'xPackMonitoring:showBanner': False},
try:
index = '.kibana'
- ES_ID = 'index-pattern:pattern-for-{}-{}'.format(
- BODY['project'], BODY['subject'])
+ ES_ID = 'index-pattern:pattern-for-{}'.format(
+ TEST_DATA_INDEX)
res = es.index(index=index, doc_type='doc',
id=ES_ID, body=INDEX_PATTERN_BODY)
p(json.dumps(INDEX_PATTERN_BODY, indent=4))
--- /dev/null
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
+##############################################################################
+# Copyright (c) 2017 Raghuram Vadapalli, Jaspreet Singh and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+"""
+This script is used to parse test logs, construct JSON BODY and push
+it to ELK DB.
+
+Usage: python push_test_data.py host:port
+
+JSON body similar to following is \
+constructed from robot files, jenkins environment
+and plot files available in workspace available post-build.
+{
+ "project": "opendaylight", <- fix string for ODL project
+ "subject": "test", <- fix string for ODL test
+ "test-type": "performance", <- if there are csv files, \
+ otherwise "functional"
+ "jenkins-silo": "releng" <- from Jenkins $SILO
+ "test-name": "openflowplugin-csit-1node-periodic \
+ -bulkomatic-perf-daily-only-carbon", <- from Jenkins $JOB_NAME
+ "test-run": 289, <- from Jenkins $BUILD_NUMBER
+ "start-time": "20170612 16:50:04 GMT-07:00", <- from robot log
+ "duration": "00:01:05.942", <- from robot log
+ "pass-tests": 9, <- from robot log
+ "fail-tests": 0, <- from robot log
+ "plots": {
+ "rate": { <- csv filename
+ "Config DS": 5816.99726601, <- from csv file
+ "OVS Switch": 5757.05238918, <- from csv file
+ "Operational DS": 2654.49139945 <- from csv file
+ },
+ "time": { <- csv filename
+ "Config DS": 17.191, <- from csv file
+ "OVS Switch": 17.37, <- from csv file
+ "Operational DS": 37.672 <- from csv file
+ }
+ }
+}
+"""
+
+# stdlib
+import json
+import sys
+
+# 3rd party lib
+from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
+
+# User defined libs
+import data_generate as data_gen
+
+
+def p(x):
+ print(json.dumps(x, indent=6, sort_keys=True))
+# ELK DB host and port to be passed as ':' separated argument
+
+
+if len(sys.argv) > 1:
+ if ':' in sys.argv[1]:
+ ELK_DB_HOST = sys.argv[1].split(':')[0]
+ ELK_DB_PORT = sys.argv[1].split(':')[1]
+else:
+ print('Usage: python push_to_elk.py host:port')
+ print('Unable to publish data to ELK. Exiting.')
+ sys.exit()
+
+# Construct json body
+
+# BODY = {}
+
+try:
+ es = Elasticsearch(
+ hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
+ scheme='https',
+ connection_class=RequestsHttpConnection
+ )
+except Exception as e:
+ print('Unexpected Error Occurred. Exiting')
+ print(e)
+# print(es.info())
+
+
+# get data from the user defined script
+BODY = data_gen.generate()
+
+print(json.dumps(BODY, indent=4))
+
+# Try to send request to ELK DB.
+
+try:
+ index = '{}-{}'.format(BODY['project'],
+ BODY['subject'])
+ ES_ID = '{}:{}-{}'.format(BODY['test-type'], BODY['test-name'],
+ BODY['test-run'])
+ res = es.index(index=index, doc_type='doc', id=ES_ID, body=BODY)
+ print(json.dumps(res, indent=4))
+except Exception as e:
+ print(e)
+ print('Unable to push data to ElasticSearch')
viz-template: opendaylight-test-performance
y-axis:
value_axes:
- 1:
- title: BGP updates/sec
- series:
1:
+ title: BGP updates/sec
+ series:
+ 2:
field: plots.performance-bgp.+10/-9 routes per UPDATE
label: +10/-9 routes per UPDATE
- color: red
+ color: blue
filter:
1:
field: test-name
viz-template: opendaylight-test-performance
y-axis:
value_axes:
- 1:
- title: BGP updates/sec
- series:
1:
+ title: BGP updates/sec
+ series:
+ 2:
field: plots.performance-bgp.+10/-9 routes per UPDATE
label: +10/-9 routes per UPDATE
- color: red
+ color: blue
filter:
1:
field: test-name
viz-template: opendaylight-test-performance
y-axis:
value_axes:
- 1:
- title: flowmods/sec
- series:
1:
+ title: flowmods/sec
+ series:
+ 2:
field: plots.rate.Config DS
label: Config DS
color: red
- 2:
+ 3:
field: plots.rate.OVS Switch
label: OVS Switch
color: green
- 3:
+ 4:
field: plots.rate.Operational DS
label: Operational DS
color: blue
viz-template: opendaylight-test-performance
y-axis:
value_axes:
- 1:
- title: flowmods/sec
- series:
1:
+ title: flowmods/sec
+ series:
+ 2:
field: plots.rate.Config DS
label: Config DS
color: red
- 2:
+ 3:
field: plots.rate.OVS Switch
label: OVS Switch
color: green
- 3:
+ 4:
field: plots.rate.Operational DS
label: Operational DS
color: blue
viz-template: opendaylight-test-performance
y-axis:
value_axes:
- 1:
- title: flowmods/sec
- series:
1:
+ title: flowmods/sec
+ series:
+ 2:
field: plots.rate.Config DS
label: Config DS
color: red
- 2:
+ 3:
field: plots.rate.OVS Switch
label: OVS Switch
color: green
- 3:
+ 4:
field: plots.rate.Operational DS
label: Operational DS
color: blue
viz-template: opendaylight-test-performance
y-axis:
value_axes:
- 1:
- title: flowmods/sec
- series:
1:
+ title: flowmods/sec
+ series:
+ 2:
field: plots.rate.Config DS
label: Config DS
color: red
- 2:
+ 3:
field: plots.rate.OVS Switch
label: OVS Switch
color: green
- 3:
+ 4:
field: plots.rate.Operational DS
label: Operational DS
color: blue
position: left
title: No. of test cases
series:
- fail:
+ 1:
data_type: histogram
mode: stacked
- field: fail-tests
- label: Fail tests
+ field: pass-tests
+ label: Pass tests
schema: metric
value_axis: 1
- color: red
- pass:
+ color: green
+ 2:
data_type: histogram
mode: stacked
- field: pass-tests
- label: Pass tests
+ field: fail-tests
+ label: Fail tests
schema: metric
value_axis: 1
- color: green
- build:
+ color: red
+ 3:
field: test-run
label: Build Number
schema: segment
mode: normal
value_axis: 1
aggs:
- build:
+ 1:
field: test-run
custom_label: Build Number
schema: segment
Code Review / integration / test.git / commitdiff