refactor the data generation code

The users should be able to write their own script to
generate the data to be pushed to kibana.

Change-Id: I33f64d0c61e6c29dcbc17d7db7c428d359b39c32
Signed-off-by: Dibya Prakash Das <dibyadascool@gmail.com>

diff --git a/csit/scripts/data_generate.py b/csit/scripts/data_generate.py
new file mode 100644 (file)
index 0000000..9a7f714
--- /dev/null
+++ b/csit/scripts/data_generate.py
@@ -0,0 +1,69 @@
+from datetime import datetime
+import glob
+import os
+import time
+import xml.etree.ElementTree as ET
+
+
+def generate():
+    BODY = {}
+
+    ts = time.time()
+    formatted_ts = \
+        datetime.fromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+    BODY['@timestamp'] = formatted_ts
+
+    # Plots are obtained from csv files ( in archives directory in $WORKSPACE).
+
+    csv_files = glob.glob('archives/*.csv')
+    BODY['project'] = 'opendaylight'
+    BODY['subject'] = 'test'
+
+    # If there are no csv files, then it is a functional test.
+    # Parse csv files and fill perfomance parameter values
+
+    if len(csv_files) == 0:
+        BODY['test-type'] = 'functional'
+    else:
+        BODY['test-type'] = 'performance'
+        BODY['plots'] = {}
+        for f in csv_files:
+            key = (f.split('/')[-1])[:-4]
+            BODY['plots'][key] = {}
+            with open(f) as file:
+                lines = file.readlines()
+            props = lines[0].strip().split(',')
+            vals = lines[1].strip().split(',')
+            BODY['plots'][key][props[0]] = float(vals[0])
+            BODY['plots'][key][props[1]] = float(vals[1])
+            BODY['plots'][key][props[2]] = float(vals[2])
+
+    # Fill the required parameters whose values are obtained from environment.
+
+    BODY['jenkins-silo'] = os.environ['SILO']
+    BODY['test-name'] = os.environ['JOB_NAME']
+    BODY['test-run'] = int(os.environ['BUILD_NUMBER'])
+
+    # Parsing robot log for stats on start-time, pass/fail tests and duration.
+
+    robot_log = os.environ['WORKSPACE'] + '/output.xml'
+    tree = ET.parse(robot_log)
+    BODY['id'] = '{}-{}'.format(os.environ['JOB_NAME'],
+                                os.environ['BUILD_NUMBER'])
+    BODY['start-time'] = tree.getroot().attrib['generated']
+    BODY['pass-tests'] = int(tree.getroot().find('statistics')
+                             [0][1].get('pass'))
+    BODY['fail-tests'] = int(tree.getroot().find('statistics')
+                             [0][1].get('fail'))
+    endtime = tree.getroot().find('suite').find('status').get('endtime')
+    starttime = tree.getroot().find('suite').find('status').get('starttime')
+    elap_time = datetime.strptime(endtime, '%Y%m%d %H:%M:%S.%f') \
+        - datetime.strptime(starttime, '%Y%m%d %H:%M:%S.%f')
+    BODY['duration'] = str(elap_time)
+
+    BODY = {
+        'type': BODY['test-type'],
+        BODY['test-type']: BODY
+    }
+
+    return BODY

diff --git a/csit/scripts/push_to_elk.py b/csit/scripts/push_to_elk.py
index 3a9ade920d2d34860bcad39ba4d3a59ef18be6e2..53cafd8c44f03839363b604c9caad3ec06463cb1 100755 (executable)
--- a/csit/scripts/push_to_elk.py
+++ b/csit/scripts/push_to_elk.py
@@ -49,13 +49,10 @@ and plot files available in workspace available post-build.
 """
 
 # stdlib
-from datetime import datetime
-import glob
 import json
 import os
 import sys
-import time
-import xml.etree.ElementTree as ET
+
 
 # 3rd party lib
 from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
@@ -64,6 +61,7 @@ import yaml
 # User defined libs
 import generate_visState as vis_gen
 import generate_dashVis as dash_gen
+import data_generate as data_gen
 
 
 def p(x):
@@ -82,7 +80,7 @@ else:
 
 # Construct json body
 
-BODY = {}
+# BODY = {}
 
 try:
     es = Elasticsearch(
@@ -95,61 +93,9 @@ except Exception as e:
     print(e)
 # print(es.info())
 
-ts = time.time()
-formatted_ts = \
-    datetime.fromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
-BODY['@timestamp'] = formatted_ts
-
-# Plots are obtained from csv files ( in archives directory in $WORKSPACE).
-
-csv_files = glob.glob('archives/*.csv')
-BODY['project'] = 'opendaylight'
-BODY['subject'] = 'test'
 
-# If there are no csv files, then it is a functional test.
-# Parse csv files and fill perfomance parameter values
-
-if len(csv_files) == 0:
-    BODY['test-type'] = 'functional'
-else:
-    BODY['test-type'] = 'performance'
-    BODY['plots'] = {}
-    for f in csv_files:
-        key = (f.split('/')[-1])[:-4]
-        BODY['plots'][key] = {}
-        with open(f) as file:
-            lines = file.readlines()
-        props = lines[0].strip().split(',')
-        vals = lines[1].strip().split(',')
-        BODY['plots'][key][props[0]] = float(vals[0])
-        BODY['plots'][key][props[1]] = float(vals[1])
-        BODY['plots'][key][props[2]] = float(vals[2])
-
-# Fill the required parameters whose values are obtained from environment.
-
-BODY['jenkins-silo'] = os.environ['SILO']
-BODY['test-name'] = os.environ['JOB_NAME']
-BODY['test-run'] = int(os.environ['BUILD_NUMBER'])
-
-# Parsing robot log for stats on start-time, pass/fail tests and duration.
-
-robot_log = os.environ['WORKSPACE'] + '/output.xml'
-tree = ET.parse(robot_log)
-BODY['id'] = '{}-{}'.format(os.environ['JOB_NAME'],
-                            os.environ['BUILD_NUMBER'])
-BODY['start-time'] = tree.getroot().attrib['generated']
-BODY['pass-tests'] = int(tree.getroot().find('statistics')[0][1].get('pass'))
-BODY['fail-tests'] = int(tree.getroot().find('statistics')[0][1].get('fail'))
-endtime = tree.getroot().find('suite').find('status').get('endtime')
-starttime = tree.getroot().find('suite').find('status').get('starttime')
-elap_time = datetime.strptime(endtime, '%Y%m%d %H:%M:%S.%f') \
-    - datetime.strptime(starttime, '%Y%m%d %H:%M:%S.%f')
-BODY['duration'] = str(elap_time)
-
-BODY = {
-    'type': BODY['test-type'],
-    BODY['test-type']: BODY
-}
+# get data from the user defined script
+BODY = data_gen.generate()
 
 print(json.dumps(BODY, indent=4))