Use graceful start and stop

Change-Id: I231ccaf155fd182d27f2b72e7532df7594de88c0
Signed-off-by: Sam Hague <shague@redhat.com>

diff --git a/csit/suites/openstack/clustering/ha_l2.robot b/csit/suites/openstack/clustering/ha_l2.robot
index 64608eba6509e039e863fe686a7bc8865896f9b9..c8830b56ce5ddc3043686e5c7a6875b8228bd330 100644 (file)
--- a/csit/suites/openstack/clustering/ha_l2.robot
+++ b/csit/suites/openstack/clustering/ha_l2.robot
@@ -65,8 +65,8 @@ Delete the Bridge Manually and Verify Before Fail
     ClusterOvsdb.Delete Sample Bridge Manually And Verify    ${OS_CNTL_IP}
 
 Take Down ODL1
-    [Documentation]    Kill the karaf in First Controller
-    ${new_cluster_list} =    ClusterManagement.Kill Single Member    1
+    [Documentation]    Stop the karaf in First Controller
+    ${new_cluster_list} =    ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
     BuiltIn.Set Suite Variable    ${new_cluster_list}
 
 Create Bridge Manually and Verify After Fail
@@ -83,7 +83,7 @@ Delete the Bridge Manually and Verify After Fail
 
 Bring Up ODL1
     [Documentation]    Bring up ODL1 again
-    ClusterManagement.Start Single Member    member=1    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    1    msg=up: ODL2, ODL3, down: ODL1
 
 Create Bridge Manually and Verify After Recover
     [Documentation]    Create bridge with OVS command and verify it gets applied from all instances.
@@ -98,8 +98,8 @@ Delete the Bridge Manually and Verify After Recover
     ClusterOvsdb.Delete Sample Bridge Manually And Verify    ${OS_CNTL_IP}
 
 Take Down ODL2
-    [Documentation]    Kill the karaf in Second Controller
-    ClusterManagement.Kill Single Member    2
+    [Documentation]    Stop the karaf in Second Controller
+    ClusterManagement.Stop Single Member    2    msg=up: ODL1, ODL2, ODL3, down=none
 
 Create Vm Instances For net_1
     [Documentation]    Create Vm instances using flavor and image names for a network.
@@ -127,7 +127,7 @@ Check Vm Instances Have Ip Address
 
 Bring Up ODL2
     [Documentation]    Bring up ODL2 again
-    ClusterManagement.Start Single Member    member=2    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    2    msg=up: ODL1, ODL3, down: ODL2
 
 Ping Vm Instance1 In net_1
     [Documentation]    Check reachability of vm instances by pinging to them.
@@ -154,8 +154,8 @@ Ping Vm Instance3 In net_2
     OpenStackOperations.Ping Vm From DHCP Namespace    @{NETWORKS}[1]    @{NET_2_VM_IPS}[2]
 
 Take Down ODL3
-    [Documentation]    Kill the karaf in Third Controller
-    ClusterManagement.Kill Single Member    3
+    [Documentation]    Stop the karaf in Third Controller
+    ClusterManagement.Stop Single Member    3    msg=up: ODL1, ODL2, ODL3, down=none
 
 Connectivity Tests From Vm Instance1 In net_1
     [Documentation]    Logging to the vm instance using generated key pair.
@@ -171,11 +171,12 @@ Connectivity Tests From Vm Instance3 In net_1
 
 Bring Up ODL3
     [Documentation]    Bring up ODL3 again
-    ClusterManagement.Start Single Member    member=3    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    3    msg=up: ODL1, ODL2, down: ODL3
 
 Take Down ODL1 and ODL2
-    [Documentation]    Kill the karaf in First and Second Controller
-    ClusterManagement.Kill Members From List Or All    ${CLUSTER_DOWN_LIST}
+    [Documentation]    Stop the karaf in First and Second Controller
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Stop Single Member    2    msg=up: ODL2, ODL3, down=ODL1
 
 Connectivity Tests From Vm Instance1 In net_2
     [Documentation]    Logging to the vm instance using generated key pair.
@@ -190,8 +191,10 @@ Connectivity Tests From Vm Instance3 In net_2
     OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_VM_IPS}[2]    ${NET_2_VM_IPS}
 
 Bring Up ODL1 and ODL2
-    [Documentation]    Bring up ODL1 and ODL2 again.
-    ClusterManagement.Start Members From List Or All    member_index_list=${CLUSTER_DOWN_LIST}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    [Documentation]    Bring up ODL1 and ODL2 again. Do not check for cluster sync until all nodes are
+    ...    up. akka will not let nodes join until they are all back up if two were down.
+    ClusterManagement.Start Single Member    1    msg=up: ODL3, down: ODL1, ODL2    wait_for_sync=False
+    ClusterManagement.Start Single Member    2    msg=up: ODL1, ODL3, down: ODL2
 
 Delete Vm Instance
     [Documentation]    Delete Vm instances using instance names. Also remove the VM from the

diff --git a/csit/suites/openstack/clustering/ha_l3.robot b/csit/suites/openstack/clustering/ha_l3.robot
index 412eb510f50c6c978572ff04906814adea358281..fe8caa4385b142a58aabe16919c330f387db17b9 100644 (file)
--- a/csit/suites/openstack/clustering/ha_l3.robot
+++ b/csit/suites/openstack/clustering/ha_l3.robot
@@ -36,10 +36,11 @@ Create All Controller Sessions
     ClusterManagement.ClusterManagement Setup
 
 Take Down Leader Of Default Shard
-    [Documentation]    Kill the karaf on ODL cluster leader
+    [Documentation]    Stop the karaf on ODL cluster leader
     ${cluster_leader}    ${followers} =    ClusterManagement.Get Leader And Followers For Shard    shard_type=config
     BuiltIn.Set Suite Variable    ${cluster_leader}
-    ClusterManagement.Kill Single Member    ${cluster_leader}
+    ${new_cluster_list} =    ClusterManagement.Stop Single Member    ${cluster_leader}    msg=up: ODL1, ODL2, ODL3, down=none
+    BuiltIn.Set Suite Variable    ${new_cluster_list}
 
 Create Networks
     [Documentation]    Create Network with neutron request.
@@ -56,15 +57,15 @@ Create Subnets For net_2
 
 Bring Up Leader Of Default Shard
     [Documentation]    Bring up on cluster leader
-    ClusterManagement.Start Single Member    member=${cluster_leader}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    ${cluster_leader}    msg=up: ${new_cluster_list}, down: ${cluster_leader}
 
 Add Ssh Allow All Rule
     [Documentation]    Allow all TCP/UDP/ICMP packets for this suite
     OpenStackOperations.Create Allow All SecurityGroup    ${SECURITY_GROUP}
 
 Take Down ODL1
-    [Documentation]    Kill the karaf in First Controller
-    ClusterManagement.Kill Single Member    1
+    [Documentation]    Stop the karaf in First Controller
+    ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
 
 Create Vm Instances For net_1
     [Documentation]    Create Vm instances using flavor and image names for a network.
@@ -74,11 +75,11 @@ Create Vm Instances For net_1
 
 Bring Up ODL1
     [Documentation]    Bring up ODL1 again
-    ClusterManagement.Start Single Member    member=1    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    1    msg=up: ODL2, ODL3, down: ODL1
 
 Take Down ODL2
-    [Documentation]    Kill the karaf in Second Controller
-    ClusterManagement.Kill Single Member    2
+    [Documentation]    Stop the karaf in Second Controller
+    ClusterManagement.Stop Single Member    2    msg=up: ODL1, ODL2, ODL3, down=none
 
 Create Vm Instances For net_2
     [Documentation]    Create Vm instances using flavor and image names for a network.
@@ -100,11 +101,11 @@ Check Vm Instances Have Ip Address
 
 Bring Up ODL2
     [Documentation]    Bring up ODL2 again
-    ClusterManagement.Start Single Member    member=2    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    2    msg=up: ODL1, ODL3, down: ODL2
 
 Take Down ODL3
-    [Documentation]    Kill the karaf in Third Controller
-    ClusterManagement.Kill Single Member    3
+    [Documentation]    Stop the karaf in Third Controller
+    ClusterManagement.Stop Single Member    3    msg=up: ODL1, ODL2, ODL3, down=none
 
 Create Router router_2
     [Documentation]    Create Router and Add Interface to the subnets.
@@ -128,7 +129,7 @@ Verify Created Routers
 
 Bring Up ODL3
     [Documentation]    Bring up ODL3 again
-    ClusterManagement.Start Single Member    member=3    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    ClusterManagement.Start Single Member    3    msg=up: ODL1, ODL2, down: ODL3
 
 Ping Vm Instance1 In net_2 From net_1
     [Documentation]    Check reachability of vm instances by pinging to them after creating routers.
@@ -155,9 +156,10 @@ Ping Vm Instance3 In net_1 From net_2
     OpenStackOperations.Ping Vm From DHCP Namespace    @{NETWORKS}[1]    @{NET_1_L3_VM_IPS}[2]
 
 Take Down ODL1 and ODL2
-    [Documentation]    Kill the karaf in First and Second Controller
-    ClusterManagement.Kill Members From List Or All    ${ODL_1_AND_2_DOWN}
-    [Teardown]    OpenStackOperations.Get OvsDebugInfo
+    [Documentation]    Stop the karaf in First and Second Controller
+    ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
+    ClusterManagement.Stop Single Member    2    msg=up: ODL2, ODL3, down=ODL1
+    [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance1 In net_1
     [Documentation]    ssh to the VM instance and test operations.
@@ -178,13 +180,17 @@ Connectivity Tests From Vm Instance3 In net_1
     [Teardown]    OpenStackOperations.Get OvsDebugInfo
 
 Bring Up ODL1 and ODL2
-    [Documentation]    Bring up ODL1 and ODL2 again
-    ClusterManagement.Start Members From List Or All    member_index_list=${ODL_1_AND_2_DOWN}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    [Documentation]    Bring up ODL1 and ODL2 again. Do not check for cluster sync until all nodes are
+    ...    up. akka will not let nodes join until they are all back up if two were down.
+    ClusterManagement.Start Single Member    1    msg=up: ODL3, down: ODL1, ODL2    wait_for_sync=False
+    ClusterManagement.Start Single Member    2    msg=up: ODL1, ODL3, down: ODL2
+    [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Take Down ODL2 and ODL3
-    [Documentation]    Kill the karaf in First and Second Controller
-    ClusterManagement.Kill Members From List Or All    ${ODL_2_AND_3_DOWN}
-    [Teardown]    OpenStackOperations.Get OvsDebugInfo
+    [Documentation]    Stop the karaf in First and Second Controller
+    ClusterManagement.Stop Single Member    2    msg=up: ODL1, ODL2, ODL3, down=none
+    ClusterManagement.Stop Single Member    3    msg=up: ODL1, ODL3, down=ODL2
+    [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance1 In net_2
     [Documentation]    ssh to the VM instance and test operations.
@@ -205,16 +211,26 @@ Connectivity Tests From Vm Instance3 In net_2
     [Teardown]    OpenStackOperations.Get OvsDebugInfo
 
 Bring Up ODL2 and ODL3
-    [Documentation]    Bring up ODL2 and ODL3 again.
-    ClusterManagement.Start Members From List Or All    member_index_list=${ODL_2_AND_3_DOWN}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    [Documentation]    Bring up ODL2 and ODL3 again. Do not check for cluster sync until all nodes are
+    ...    up. akka will not let nodes join until they are all back up if two were down.
+    ClusterManagement.Start Single Member    2    msg=up: ODL1, down: ODL2, ODL3    wait_for_sync=False
+    ClusterManagement.Start Single Member    3    msg=up: ODL1, ODL2, down: ODL3
+    [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Take Down All Instances
     [Documentation]    Stop karaf on all controllers
-    ClusterManagement.Kill Members From List Or All
+    ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
+    ClusterManagement.Stop Single Member    2    msg=up: ODL2, ODL3, down=ODL1
+    ClusterManagement.Stop Single Member    3    msg=up: ODL3, down=ODL1, ODL2
+    [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Bring Up All Instances
-    [Documentation]    Bring up all controllers
-    ClusterManagement.Start Members From List Or All    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
+    [Documentation]    Bring up all controllers. Do not check for cluster sync until all nodes are
+    ...    up. akka will not let nodes join until they are all back up if two were down.
+    ClusterManagement.Start Single Member    1    msg=up: none, down: ODL1, ODL2, ODL3    wait_for_sync=False
+    ClusterManagement.Start Single Member    2    msg=up: ~ODL1, down: ODL2, ODL3    wait_for_sync=False
+    ClusterManagement.Start Single Member    3    msg=up: ~ODL1, ~ODL2, down: ODL3
+    [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance2 In net_2 after recovering all nodes
     [Documentation]    ssh to the VM instance and test operations.