Add tools to isolate controllers.

Change-Id: I367c2a00a32d7a84d6468ee590322bc7512a4d15
Signed-off-by: Phillip Shea <phillip.shea@hp.com>

diff --git a/test/csit/libraries/UtilLibrary.py b/test/csit/libraries/UtilLibrary.py
index 92284d67b4bb7f17d8aa86ef2a4dc97345955bff..05ef69ab374ada992473714c97b64244f737328d 100644 (file)
--- a/test/csit/libraries/UtilLibrary.py
+++ b/test/csit/libraries/UtilLibrary.py
@@ -208,13 +208,13 @@ def isolate_controller(controllers, username, password, isolated):
     :return: If successful, returns "pass", otherwise returns the last failed IPTables text.
     """
     isolated_controller = controllers[isolated-1]
     :return: If successful, returns "pass", otherwise returns the last failed IPTables text.
     """
     isolated_controller = controllers[isolated-1]

-    del controllers[isolated-1]

     for controller in controllers:
     for controller in controllers:

-        base_str = 'sudo iptables -I OUTPUT -p all --source '
-        cmd_str = base_str + isolated_controller + ' --destination ' + controller + ' -j DROP'
-        execute_ssh_command(isolated_controller, username, password, cmd_str)
-        cmd_str = base_str + controller + ' --destination ' + isolated_controller + ' -j DROP'
-        execute_ssh_command(isolated_controller, username, password, cmd_str)
+        if controller != isolated_controller:
+            base_str = 'sudo iptables -I OUTPUT -p all --source '
+            cmd_str = base_str + isolated_controller + ' --destination ' + controller + ' -j DROP'
+            execute_ssh_command(isolated_controller, username, password, cmd_str)
+            cmd_str = base_str + controller + ' --destination ' + isolated_controller + ' -j DROP'
+            execute_ssh_command(isolated_controller, username, password, cmd_str)

     ip_tables = execute_ssh_command(isolated_controller, username, password, 'sudo iptables -L')
     print ip_tables
     iso_result = 'pass'
     ip_tables = execute_ssh_command(isolated_controller, username, password, 'sudo iptables -L')
     print ip_tables
     iso_result = 'pass'


@@ -240,13 +240,13 @@ def rejoin_controller(controllers, username, password, isolated):
     :return: If successful, returns "pass", otherwise returns the last failed IPTables text.
     """
     isolated_controller = controllers[isolated-1]
     :return: If successful, returns "pass", otherwise returns the last failed IPTables text.
     """
     isolated_controller = controllers[isolated-1]

-    del controllers[isolated-1]

     for controller in controllers:
     for controller in controllers:

-        base_str = 'sudo iptables -D OUTPUT -p all --source '
-        cmd_str = base_str + isolated_controller + ' --destination ' + controller + ' -j DROP'
-        execute_ssh_command(isolated_controller, username, password, cmd_str)
-        cmd_str = base_str + controller + ' --destination ' + isolated_controller + ' -j DROP'
-        execute_ssh_command(isolated_controller, username, password, cmd_str)
+        if controller != isolated_controller:
+            base_str = 'sudo iptables -D OUTPUT -p all --source '
+            cmd_str = base_str + isolated_controller + ' --destination ' + controller + ' -j DROP'
+            execute_ssh_command(isolated_controller, username, password, cmd_str)
+            cmd_str = base_str + controller + ' --destination ' + isolated_controller + ' -j DROP'
+            execute_ssh_command(isolated_controller, username, password, cmd_str)

     ip_tables = execute_ssh_command(isolated_controller, username, password, 'sudo iptables -L')
     print ip_tables
     iso_result = 'pass'
     ip_tables = execute_ssh_command(isolated_controller, username, password, 'sudo iptables -L')
     print ip_tables
     iso_result = 'pass'


@@ -272,6 +272,7 @@ def flush_iptables(controllers, username, password):
     """
     flush_result = 'pass'
     for controller in controllers:
     """
     flush_result = 'pass'
     for controller in controllers:

+        print 'Flushing ' + controller

         cmd_str = 'sudo iptables -v -F'
         cmd_result = execute_ssh_command(controller, username, password, cmd_str)
         print cmd_result
         cmd_str = 'sudo iptables -v -F'
         cmd_result = execute_ssh_command(controller, username, password, cmd_str)
         print cmd_result

diff --git a/test/tools/clustering/cluster-monitor/cluster.json b/test/tools/clustering/cluster-monitor/cluster.json
new file mode 100644 (file)
index 0000000..299d8b4
--- /dev/null
+++ b/test/tools/clustering/cluster-monitor/cluster.json
@@ -0,0 +1,11 @@
+{
+    "cluster": {
+        "controllers": [
+            "172.17.10.93",
+            "172.17.10.94",
+            "172.17.10.95"
+        ],
+        "user": "username",
+        "pass": "password"
+    }
+}

diff --git a/test/tools/clustering/cluster-monitor/isolate.py b/test/tools/clustering/cluster-monitor/isolate.py
new file mode 100644 (file)
index 0000000..d5960a9
--- /dev/null
+++ b/test/tools/clustering/cluster-monitor/isolate.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+"""
+Controller Isolation Tool
+Author: Phillip Shea
+Updated: 2015-May-07
+
+This tool isolates an indicated controller from the cluster. The tool takes
+a single integer argument corresponding to the number of a controller
+in a json file's ordered list of controllers. This is the controller to
+be isolated.
+
+A file named 'cluster.json' containing a list of the IP addresses and
+credentials of the controllers is required. It resides in the same
+directory as monitor.py.
+
+The file should look like this:
+
+    {
+        "cluster": {
+            "controllers": [
+                "172.17.10.93",
+                "172.17.10.94",
+                "172.17.10.95"
+            ],
+            "user": "username",
+            "pass": "password"
+        }
+    }
+
+Usage:python isolate.py [controller to be isolated]
+"""
+
+import sys
+sys.path.append('../../../csit/libraries')
+import UtilLibrary
+import json
+
+try:
+    with open('cluster.json') as cluster_file:
+        data = json.load(cluster_file)
+except:
+    print str(sys.exc_info())
+    print "unable to open the file cluster.json"
+    exit(1)
+try:
+    cluster_list = data["cluster"]["controllers"]
+    user_name = data["cluster"]["user"]
+    user_pass = data["cluster"]["pass"]
+except:
+    print str(sys.exc_info())
+    print 'Error reading the file cluster.json'
+    exit(1)
+try:
+    isolate = int(sys.argv[1])
+except:
+    print 'You must specify the number (e.g. 1, 2, 3) of the controller to isolate.'
+    exit(1)
+
+print "isolating controller " + str(isolate)
+
+print UtilLibrary.isolate_controller(cluster_list, user_name, user_pass, isolate)

diff --git a/test/tools/clustering/cluster-monitor/monitor.py b/test/tools/clustering/cluster-monitor/monitor.py
index 119b5494e27e1d4808dde19a71c89f4bbac10b89..8cafbf5763fdac709756d44665968c3678c141c5 100644 (file)
--- a/test/tools/clustering/cluster-monitor/monitor.py
+++ b/test/tools/clustering/cluster-monitor/monitor.py
@@ -2,15 +2,30 @@
 """
 Cluster Monitor Tool
 Author: Phillip Shea
 """
 Cluster Monitor Tool
 Author: Phillip Shea

-Updated: 2015-04-29
-
-This tool provides real-time monitoring of the cluster member roles for all shards.
-It is assumed that all cluster members have the same shards.
-
-Usage:python monitor.py [controller IP1] [controller IP2] ... [controller IPn]
-
-Before using, start and configure all controllers in the cluster.
-The cluster deployer script is recommended.
+Updated: 2015-May-07
+
+This tool provides real-time visualization of the cluster member roles for all
+shards in the config datastore. It is assumed that all cluster members have the
+same shards.
+
+A file named 'cluster.json' contaning a list of the IP addresses of the
+controllers is required. This resides in the same directory as monitor.py.
+"user" and "pass" are not required for monitor.py, but they may be
+needed for other apps in this folder. The file should look like this:
+
+    {
+        "cluster": {
+            "controllers": [
+                "172.17.10.93",
+                "172.17.10.94",
+                "172.17.10.95"
+            ],
+            "user": "username",
+            "pass": "password"
+        }
+    }
+
+Usage:python monitor.py

 """
 from io import BytesIO
 import time
 """
 from io import BytesIO
 import time


@@ -79,14 +94,18 @@ def size_and_color(cluster_roles, field_length, ip_addr):
 
 
 field_len = 14
 
 
 field_len = 14

-controllers = sys.argv[1:]
-print ''
-print controllers
-if len(controllers) == 0:
-    print 'Error'
-    print 'You must specify at least one controller at the command line.'
-    print 'e.g.: $python monitor.py 172.17.10.93 172.17.10.94 172.17.10.95'
-    print ' '
+try:
+    with open('cluster.json') as cluster_file:
+        data = json.load(cluster_file)
+except:
+    print str(sys.exc_info())
+    print 'Unable to open the file cluster.json'
+    exit(1)
+try:
+    controllers = data["cluster"]["controllers"]
+except:
+    print str(sys.exc_info())
+    print 'Error reading the file cluster.json'

     exit(1)
 
 controller_names = []
     exit(1)
 
 controller_names = []


@@ -97,8 +116,8 @@ for controller in controllers:
     try:
         data = rest_get(url)
     except:
     try:
         data = rest_get(url)
     except:

-        print "Unable to retrieve shard names from " + controller
-        print "Are all controllers up?"
+        print 'Unable to retrieve shard names from ' + controller
+        print 'Are all controllers up?'

         print str(sys.exc_info()[1])
         exit(1)
     print 'shards from the first controller'
         print str(sys.exc_info()[1])
         exit(1)
     print 'shards from the first controller'

diff --git a/test/tools/clustering/cluster-monitor/readme.txt b/test/tools/clustering/cluster-monitor/readme.txt
new file mode 100644 (file)
index 0000000..b0cceca
--- /dev/null
+++ b/test/tools/clustering/cluster-monitor/readme.txt
@@ -0,0 +1,75 @@
+Cluster Monitor Tool
+Author: Phillip Shea
+Updated: 2015-May-07
+
+This tool provides real-time visualization of the cluster member roles for all
+shards in the config datastore. It is useful for understanding cluster behavior
+in when controllers are isolated, downed, or rebooted. The tool assumes that all
+cluster members have the same shards.
+
+A file named 'cluster.json' containg a list of the IP addresses of the
+controllers is required. This resides in the same directory as monitor.py.
+"user" and "pass" are not required for monitor.py, but they may be
+needed for other apps in this folder. Because the contains configuration
+information unique to your environment, it may be more convenient to
+copy the contents of this folder out of git to prevent these settings
+from being overwritten by updates.
+
+
+The file should look like this:
+
+    {
+        "cluster": {
+            "controllers": [
+                "172.17.10.93",
+                "172.17.10.94",
+                "172.17.10.95"
+            ],
+            "user": "username",
+            "pass": "password"
+        }
+    }
+
+Usage:python monitor.py
+
+Starting:
+
+Before using, start and configure all controllers in the cluster. Use of the
+cluster deployer script is recommended. All controllers must initially be
+running so the tool can retrieve the controller and shard names. Once
+the tool is started and the controller and cluster shard names are retrieved,
+controllers can be isolated, downed, rebooted, etc.
+
+
+The UI:
+
+Controller member names (not host names) are displayed across the top. Shard
+names are displayed to the left.
+
+In the upper left is a heart emoticon "<3" which toggles between yellow and
+black backgrounds with each update. If a controller is down, the http timeout
+comes in to play and updating becomes much slower.
+
+The central matrix displays controller roles. When rest queries fail, the
+error type is displayed. Leader, Follower, and Candidate roles are color-
+coded.
+
+
+Other scripts:
+
+isolate.py
+
+    Isolates an indicated controller from the cluster.
+
+rejoin.py
+
+    Rejoins any isolated controllers to the cluster.
+
+timed_isolation.py
+
+    Isolates an indicated controller for a specified duration
+
+
+Future enhancements:
+
+    Add operational shards.
\ No newline at end of file

diff --git a/test/tools/clustering/cluster-monitor/rejoin.py b/test/tools/clustering/cluster-monitor/rejoin.py
new file mode 100644 (file)
index 0000000..92507a0
--- /dev/null
+++ b/test/tools/clustering/cluster-monitor/rejoin.py
@@ -0,0 +1,51 @@
+#!/usr/bin/python
+"""
+Cluster Rejoin Tool
+Author: Phillip Shea
+Updated: 2015-May-07
+
+This tool rejoins any isolated controllers to the cluster.
+
+A file named 'cluster.json' containing a list of the IP addresses and
+credentials of the controllers is required. It resides in the same
+directory as monitor.py.
+
+The file should look like this:
+
+    {
+        "cluster": {
+            "controllers": [
+                "172.17.10.93",
+                "172.17.10.94",
+                "172.17.10.95"
+            ],
+            "user": "username",
+            "pass": "password"
+        }
+    }
+
+Usage:python rejoin.py
+ """
+
+import sys
+sys.path.append('../../../csit/libraries')
+import UtilLibrary
+import json
+
+try:
+    with open('cluster.json') as cluster_file:
+        data = json.load(cluster_file)
+except:
+    print str(sys.exc_info())
+    print "unable to open the file cluster.json"
+    exit(1)
+try:
+    cluster_list = data["cluster"]["controllers"]
+    user_name = data["cluster"]["user"]
+    user_pass = data["cluster"]["pass"]
+except:
+    print str(sys.exc_info())
+    print 'Error reading the file cluster.json'
+    exit(1)
+
+print UtilLibrary.flush_iptables(cluster_list, user_name, user_pass)

diff --git a/test/tools/clustering/cluster-monitor/timed_isolation.py b/test/tools/clustering/cluster-monitor/timed_isolation.py
new file mode 100644 (file)
index 0000000..8492575
--- /dev/null
+++ b/test/tools/clustering/cluster-monitor/timed_isolation.py
@@ -0,0 +1,69 @@
+#!/usr/bin/python
+"""
+Controller Isolation Tool
+Author: Phillip Shea
+Updated: 2015-May-07
+
+This tool isolates an indicated controller for a specified duration.
+The tool's first integer argument corresponds to the number of a controller
+in a json file's ordered list of controllers. This is the controller to
+be isolated. The second argument is the duration of isolation in
+seconds.
+
+A file named 'cluster.json' containing a list of the IP addresses and
+credentials of the controllers is required. It resides in the same
+directory as monitor.py.
+
+The file should look like this:
+
+    {
+        "cluster": {
+            "controllers": [
+                "172.17.10.93",
+                "172.17.10.94",
+                "172.17.10.95"
+            ],
+            "user": "username",
+            "pass": "password"
+        }
+    }
+
+Usage:python timed_isolation.py [controller to be isolated]  [duration of isolation in seconds]
+"""
+
+import sys
+sys.path.append('../../../csit/libraries')
+import UtilLibrary
+import json
+import time
+
+try:
+    with open('cluster.json') as cluster_file:
+        data = json.load(cluster_file)
+except:
+    print str(sys.exc_info())
+    print 'unable to open the file cluster.json'
+    exit(1)
+try:
+    controllers = data["cluster"]["controllers"]
+    user_name = data["cluster"]["user"]
+    user_pass = data["cluster"]["pass"]
+except:
+    print str(sys.exc_info())
+    print 'Error reading the file cluster.json'
+    exit(1)
+try:
+    isolate = int(sys.argv[1])
+    duration = int(sys.argv[2])
+except:
+    print 'You must specify the number (e.g. 1, 2, 3) of the controller to isolate.'
+    exit(1)
+
+print 'Isolating controller ' + str(isolate)
+
+print UtilLibrary.isolate_controller(controllers, user_name, user_pass, isolate)
+
+print 'Pausing for ' + str(duration) + ' seconds...'
+time.sleep(duration)
+
+print UtilLibrary.flush_iptables(controllers, user_name, user_pass)