csit/scripts/data_generate.py

   1 from datetime import datetime
   2 import glob
   3 import os
   4 import time
   5 import xml.etree.ElementTree as ET
   6
   7
   8 def generate():
   9     BODY = {}
  10
  11     ts = time.time()
  12     formatted_ts = \
  13         datetime.fromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
  14     BODY['@timestamp'] = formatted_ts
  15
  16     # Plots are obtained from csv files ( in archives directory in $WORKSPACE).
  17
  18     csv_files = glob.glob('archives/*.csv')
  19     BODY['project'] = 'opendaylight'
  20     BODY['subject'] = 'test'
  21
  22     # If there are no csv files, then it is a functional test.
  23     # Parse csv files and fill perfomance parameter values
  24
  25     if len(csv_files) == 0:
  26         BODY['test-type'] = 'functional'
  27     else:
  28         BODY['test-type'] = 'performance'
  29         BODY['plots'] = {}
  30         for f in csv_files:
  31             key = (f.split('/')[-1])[:-4]
  32             BODY['plots'][key] = {}
  33             with open(f) as file:
  34                 lines = file.readlines()
  35             props = lines[0].strip().split(',')
  36             vals = lines[1].strip().split(',')
  37             BODY['plots'][key][props[0]] = float(vals[0])
  38             BODY['plots'][key][props[1]] = float(vals[1])
  39             BODY['plots'][key][props[2]] = float(vals[2])
  40
  41     # Fill the required parameters whose values are obtained from environment.
  42
  43     BODY['jenkins-silo'] = os.environ['SILO']
  44     BODY['test-name'] = os.environ['JOB_NAME']
  45     BODY['test-run'] = int(os.environ['BUILD_NUMBER'])
  46
  47     # Parsing robot log for stats on start-time, pass/fail tests and duration.
  48
  49     robot_log = os.environ['WORKSPACE'] + '/output.xml'
  50     tree = ET.parse(robot_log)
  51     BODY['id'] = '{}-{}'.format(os.environ['JOB_NAME'],
  52                                 os.environ['BUILD_NUMBER'])
  53     BODY['start-time'] = tree.getroot().attrib['generated']
  54     BODY['pass-tests'] = int(tree.getroot().find('statistics')
  55                              [0][1].get('pass'))
  56     BODY['fail-tests'] = int(tree.getroot().find('statistics')
  57                              [0][1].get('fail'))
  58     endtime = tree.getroot().find('suite').find('status').get('endtime')
  59     starttime = tree.getroot().find('suite').find('status').get('starttime')
  60     elap_time = datetime.strptime(endtime, '%Y%m%d %H:%M:%S.%f') \
  61         - datetime.strptime(starttime, '%Y%m%d %H:%M:%S.%f')
  62     BODY['duration'] = str(elap_time)
  63
  64     BODY = {
  65         'type': BODY['test-type'],
  66         BODY['test-type']: BODY
  67     }
  68
  69     return BODY