SXP: Check cluster is ready after (un-)isolation

JIRA: SXP-150
Change-Id: I5e587775054a6da4ee392439727abdf0ab64d3e6
Signed-off-by: Ivan Hrasko <ihrasko@cisco.com>

diff --git a/csit/suites/sxp/cluster-routing/010_Route_Switching.robot b/csit/suites/sxp/cluster-routing/010_Route_Switching.robot
index cbb4207e7f2480e68da8faec32e6747f6ee95fcc..c2717a2a96fa8c9ce0fa5711d384790ff959bca3 100644 (file)
--- a/csit/suites/sxp/cluster-routing/010_Route_Switching.robot
+++ b/csit/suites/sxp/cluster-routing/010_Route_Switching.robot
@@ -70,10 +70,13 @@ Isolate SXP Controller
     [Documentation]    Isolate the cluster leader node and perform check that virtual IP is routed to a new leader,
     ...    afterwards unisolate old leader.
     @{running_members} =    ClusterManagement.Isolate_Member_From_List_Or_All    ${controller_index}
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${running_members}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Sync_Status_Should_Be_False    ${controller_index}
     BuiltIn.Wait Until Keyword Succeeds    240    1    SxpClusterLib.Ip Addres Should Not Be Routed To Follower    ${MAC_ADDRESS_TABLE}    ${VIRTUAL_IP}    ${controller_index}
     ${running_member} =    Collections.Get From List    ${running_members}    0
     ${active_follower} =    SxpClusterLib.Get Owner Controller    ${running_member}
     BuiltIn.Wait Until Keyword Succeeds    240    1    SxpClusterLib.Ip Addres Should Be Routed To Follower    ${MAC_ADDRESS_TABLE}    ${VIRTUAL_IP}    ${active_follower}
     ClusterManagement.Flush_Iptables_From_List_Or_All
-    BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Check_Cluster_Is_In_Sync
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${EMPTY}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}

diff --git a/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot b/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot
index edb515ece485e7ff4cd81b4a5881fd20c0693155..01e7e347ee30a178a89e7d48f688fa4a49c01900 100644 (file)
--- a/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot
+++ b/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot
@@ -104,7 +104,8 @@ Clean Custom SXP Cluster Session
 Clean Custom SXP Cluster
     [Documentation]    Disconnect SXP cluster topology
     ClusterManagement.Flush_Iptables_From_List_Or_All
-    BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Check_Cluster_Is_In_Sync
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${EMPTY}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     SxpLib.Delete Node    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
     SxpLib.Delete Node    ${CLUSTER_NODE_ID}    session=${CONTROLLER_SESSION}
 
@@ -118,6 +119,8 @@ Isolate SXP Controller
     [Arguments]    ${controller_index}
     [Documentation]    Isolate cluster node specified by ${controller_index} and find new owner
     @{running_members} =    ClusterManagement.Isolate_Member_From_List_Or_All    ${controller_index}
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${running_members}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Sync_Status_Should_Be_False    ${controller_index}
     BuiltIn.Wait Until Keyword Succeeds    240    1    SxpClusterLib.Ip Addres Should Not Be Routed To Follower    ${MAC_ADDRESS_TABLE}    ${VIRTUAL_IP}    ${controller_index}
     ${running_member} =    Collections.Get From List    ${running_members}    0

diff --git a/csit/suites/sxp/clustering/010_Connection_switchover.robot b/csit/suites/sxp/clustering/010_Connection_switchover.robot
index 8aeac109baf07aef8892ce95022cbe65f4488a91..54b9017fee37d2e385fd30e66557b0456e4ce3a8 100644 (file)
--- a/csit/suites/sxp/clustering/010_Connection_switchover.robot
+++ b/csit/suites/sxp/clustering/010_Connection_switchover.robot
@@ -24,9 +24,12 @@ Isolation of SXP noservice follower Test
 Isolate SXP Controller
     [Arguments]    ${controller_index}
     [Documentation]    Isolate one of cluster nodes and perform check that Device is still connected afterwards reverts isolation
-    ClusterManagement.Isolate_Member_From_List_Or_All    ${controller_index}
+    ${running_members} =    ClusterManagement.Isolate_Member_From_List_Or_All    ${controller_index}
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${running_members}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Sync_Status_Should_Be_False    ${controller_index}
     BuiltIn.Wait Until Keyword Succeeds    60    1    SxpClusterLib.Check Device is Connected    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
     ClusterManagement.Flush_Iptables_From_List_Or_All
-    BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Check_Cluster_Is_In_Sync
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${EMPTY}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    60    1    SxpClusterLib.Check Device is Connected    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}

diff --git a/csit/suites/sxp/clustering/030_RPC_functionality.robot b/csit/suites/sxp/clustering/030_RPC_functionality.robot
index 111d56b07f203936661f07d5a4a7bc948b271e53..cd89d91b0399042d4b6a7c4f39d8beaa14b32dd1 100644 (file)
--- a/csit/suites/sxp/clustering/030_RPC_functionality.robot
+++ b/csit/suites/sxp/clustering/030_RPC_functionality.robot
@@ -30,6 +30,8 @@ Isolate SXP Controller
     : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
     \    SxpLib.Add Bindings    ${i+1}0    ${i+1}0.${i+1}0.${i+1}0.${i+1}0/32    node=${INADDR_ANY}    session=ClusterManagement__session_${owner_controller}
     @{running_members} =    ClusterManagement.Isolate_Member_From_List_Or_All    ${controller_index}
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${running_members}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Sync_Status_Should_Be_False    ${controller_index}
     ${running_member} =    Collections.Get From List    ${running_members}    0
     ${owner_controller} =    SxpClusterLib.Get Owner Controller    ${running_member}
@@ -37,7 +39,8 @@ Isolate SXP Controller
     : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
     \    SxpLib.Delete Bindings    ${i+1}0    ${i+1}0.${i+1}0.${i+1}0.${i+1}0/32    node=${INADDR_ANY}    session=ClusterManagement__session_${owner_controller}
     ClusterManagement.Flush_Iptables_From_List_Or_All
-    BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Check_Cluster_Is_In_Sync
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${EMPTY}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    30    1    Check Bindings Does Not Exist    ${owner_controller}
 
 Check Bindings Exist

diff --git a/csit/suites/sxp/clustering/040_Bindings_consistency.robot b/csit/suites/sxp/clustering/040_Bindings_consistency.robot
index 1766bac250ebde51822c4ebbc878a8012ec690b2..30d56c16534a39d3f82e1c8b65466f5fcea5af38 100644 (file)
--- a/csit/suites/sxp/clustering/040_Bindings_consistency.robot
+++ b/csit/suites/sxp/clustering/040_Bindings_consistency.robot
@@ -67,6 +67,8 @@ Isolate SXP Controller
     [Arguments]    ${controller_index}    ${node}    ${session}=ClusterManagement__session_${controller_index}
     [Documentation]    Isolate one of cluster nodes and perform check that device is connected
     @{running_members} =    ClusterManagement.Isolate_Member_From_List_Or_All    ${controller_index}
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${running_members}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     ${running_member} =    Collections.Get From List    ${running_members}    0
     BuiltIn.Set Test Variable    ${RUNNING_MEMBER}    ${running_member}
     BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Sync_Status_Should_Be_False    ${controller_index}
@@ -76,7 +78,8 @@ UnIsolate SXP Controller
     [Arguments]    ${controller_index}
     [Documentation]    Un-Isolate one of cluster nodes and perform check that device is connected
     ClusterManagement.Flush_Iptables_From_List_Or_All
-    BuiltIn.Wait Until Keyword Succeeds    240    1    ClusterManagement.Check_Cluster_Is_In_Sync
+    BuiltIn.Wait_Until_Keyword_Succeeds    60    1    ClusterManagement.Verify_Members_Are_Ready    member_index_list=${EMPTY}    verify_cluster_sync=True    verify_restconf=True
+    ...    verify_system_status=False    service_list=@{EMPTY}
     BuiltIn.Wait Until Keyword Succeeds    60    1    SxpClusterLib.Check Device is Connected    ${DEVICE_NODE_ID}    session=${DEVICE_SESSION}
 
 Check Bindings