use Elasticsearch python client

make errors more robust and verbose

Change-Id: I385f3e41c6db1187280df3609be6c0d4b19f021e
Signed-off-by: Dibya Prakash Das <dibyadascool@gmail.com>

diff --git a/csit/postplans/openflowplugin-perf-bulkomatic.txt b/csit/postplans/openflowplugin-perf-bulkomatic.txt
index 2ff7e1dfb4b6dbd354c9d37d0f2aa8f30a7fab93..5ccfa18c00f6f7bf399c163ded281956efef50fe 100644 (file)
--- a/csit/postplans/openflowplugin-perf-bulkomatic.txt
+++ b/csit/postplans/openflowplugin-perf-bulkomatic.txt
@@ -1,2 +1,2 @@
 # Place the commands in run order:
-python integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243
+python3 integration/test/csit/scripts/push_to_elk.py a4ff38b99ef2c7626450543021b4c134.us-east-1.aws.found.io:9243

diff --git a/csit/scripts/push_to_elk.py b/csit/scripts/push_to_elk.py
index e0d37ffd111fb09cd0e665b7158b042ab85835ed..da6db610e42c67149072db6e304924f2a6321a13 100755 (executable)
--- a/csit/scripts/push_to_elk.py
+++ b/csit/scripts/push_to_elk.py
@@ -50,12 +50,12 @@ from datetime import datetime
 import glob
 import json
 import os
-import requests
 import sys
 import time
 import xml.etree.ElementTree as ET
 
 # 3rd party lib
+from elasticsearch import Elasticsearch, RequestsHttpConnection, exceptions
 import yaml
 
 # ELK DB host and port to be passed as ':' separated argument
@@ -73,6 +73,17 @@ else:
 
 BODY = {}
 
+try:
+    es = Elasticsearch(
+        hosts=[{'host': ELK_DB_HOST, 'port': int(ELK_DB_PORT)}],
+        connection_class=RequestsHttpConnection
+    )
+except Exception as e:
+    print('Unexpected Error Occurred. Exiting')
+    print(e)
+
+print(es.info())
+
 ts = time.time()
 formatted_ts = \
     datetime.fromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
@@ -124,29 +135,23 @@ elap_time = datetime.strptime(endtime, '%Y%m%d %H:%M:%S.%f') \
     - datetime.strptime(starttime, '%Y%m%d %H:%M:%S.%f')
 BODY['duration'] = str(elap_time)
 
-# Parse JSON BODY to construct PUT_URL
-
-PUT_URL_INDEX = '/{}-{}'.format(BODY['project'], BODY['subject'])
-PUT_URL_TYPE = '/{}'.format(BODY['test-type'])
-PUT_URL_ID = '/{}-{}'.format(BODY['test-name'], BODY['test-run'])
-PUT_URL = 'https://{}:{}{}{}{}'.format(ELK_DB_HOST, ELK_DB_PORT, PUT_URL_INDEX, PUT_URL_TYPE, PUT_URL_ID)
-print(PUT_URL)
-
 print(json.dumps(BODY, indent=4))
 
 # Try to send request to ELK DB.
 
 try:
-    r = requests.put(PUT_URL, json=BODY)
-    print(r.status_code)
-    print(json.dumps(r.json(), indent=4))
-except:
+    index = '{}-{}'.format(BODY['project'], BODY['subject'])
+    ES_ID = '{}-{}'.format(BODY['test-name'], BODY['test-run'])
+    res = es.index(index=index, doc_type=BODY['test-type'], id=ES_ID, body=BODY)
+    print(json.dumps(res, indent=4))
+except Exception as e:
+    print(e)
     print('Unable to push data to ElasticSearch')
 
-
 # Function to convert JSON object to string.
 # Python puts 'true' as 'True' etc. which need handling.
 
+
 def JSONToString(jobj):
     retval = str(jobj)
     retval = retval.replace('\'', '"')
@@ -314,28 +319,27 @@ if BODY['test-type'] == 'performance':
             fieldlist.append('plots.' + key + '.' + subkey)
         vis = getVisualization(BODY['test-name'], fieldlist, key)
         vis_ids.append(BODY['test-name'] + '-' + key)
-        PUT_URL = \
-            'https://{}:{}/.kibana/visualization/{}-{}'.format(ELK_DB_HOST, ELK_DB_PORT, BODY['test-name'], key)
-        print(PUT_URL)
         print(json.dumps(vis, indent=4))
         try:
-            r = requests.put(PUT_URL, json=vis)
-            print(r.status_code)
-            print(json.dumps(r.json(), indent=4))
-        except:
+            ES_ID = '{}-{}'.format(BODY['test-name'], key)
+            res = es.index(index='.kibana', doc_type='visualization', id=ES_ID, body=BODY)
+            print(json.dumps(res, indent=4))
+        except Exception as e:
+            print(e)
             print('Unable to push visualization to Kibana')
 
+
 vis = getVisualization(BODY['test-name'],
                        ['pass-tests', 'failed-tests'])
 vis_ids.append(BODY['test-name'])
-PUT_URL = 'https://{}:{}/.kibana/visualization/{}'.format(ELK_DB_HOST, ELK_DB_PORT, BODY['test-name'])
-print(PUT_URL)
+
 print(json.dumps(vis, indent=4))
 try:
-    r = requests.put(PUT_URL, json=vis)
-    print(r.status_code)
-    print(json.dumps(r.json(), indent=4))
-except:
+    ES_ID = BODY['test-name']
+    res = es.index(index='.kibana', doc_type='visualization', id=ES_ID, body=vis)
+    print(json.dumps(res, indent=4))
+except Exception as e:
+    print(e)
     print('Unable to push dashboard to Kibana')
 
 # Create dashboard and add above created visualizations to it
@@ -356,16 +360,25 @@ dashboard['kibanaSavedObjectMeta'] = {
 
 # Check if visualizations already present in dashboard. If present, don't add, else, add at end
 
-GET_URL = 'https://{}:{}/.kibana/dashboard/{}'.format(ELK_DB_HOST, ELK_DB_PORT, DASHBOARD_NAME)
-r = requests.get(GET_URL)
-response = r.json()
-dashboard_found = response['found']
+ES_ID = DASHBOARD_NAME
+try:
+    res = es.get(index='.kibana', doc_type='dashboard', id=ES_ID)
+    print(json.dumps(res, indent=4))
+    # No exeception occured means dashboard found
+    dashboard_found = True
+except exceptions.NotFoundError as e:
+    print('No visualizations found')
+    dashboard_found = False
+except Exception as e:
+    print(e)
+    print('Error Occurred')
+
 vis_ids_present = set()
 panelsJSON = []
 
-print(json.dumps(response, indent=4))
+
 if dashboard_found:
-    panelsJSON = yaml.safe_load(response['_source']['panelsJSON'])
+    panelsJSON = yaml.safe_load(res['_source']['panelsJSON'])
     for vis in panelsJSON:
         vis_ids_present.add(vis['id'])
 
@@ -393,9 +406,16 @@ for (i, vis_id) in enumerate(vis_ids):
         print('visualization ' + vis_id + ' already present in dashboard')
 
 dashboard['panelsJSON'] = JSONToString(panelsJSON)
-PUT_URL = 'https://{}:{}/.kibana/dashboard/{}'.format(ELK_DB_HOST, ELK_DB_PORT, DASHBOARD_NAME)
-print(PUT_URL)
+
 print(json.dumps(dashboard, indent=4))
-r = requests.put(PUT_URL, json=dashboard)
-print(r.status_code)
-print(json.dumps(r.json(), indent=4))
+
+try:
+    ES_ID = DASHBOARD_NAME
+    res = es.index(index='.kibana', doc_type='dashboard', id=ES_ID, body=dashboard)
+    print(json.dumps(res, indent=4))
+except exceptions.TransportError as et:
+    print(et)
+    print('Elasticsearch returned an error')
+except Exception as e:
+    print(e)
+    print('Unexpected error occurred. Unable to push dashboard to Kibana')