From 5b3c5c1ec477658eae6661282bd65f8baf94492d Mon Sep 17 00:00:00 2001 From: Ivan Hrasko Date: Wed, 9 Jan 2019 15:42:18 +0100 Subject: [PATCH] SXP: Check cluster is ready after (un-)isolation JIRA: SXP-150 Change-Id: I5e587775054a6da4ee392439727abdf0ab64d3e6 Signed-off-by: Ivan Hrasko --- csit/suites/sxp/cluster-routing/010_Route_Switching.robot | 5 ++++- .../sxp/cluster-routing/020_Sxp_Node_Switching.robot | 5 ++++- csit/suites/sxp/clustering/010_Connection_switchover.robot | 7 +++++-- csit/suites/sxp/clustering/030_RPC_functionality.robot | 5 ++++- csit/suites/sxp/clustering/040_Bindings_consistency.robot | 5 ++++- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/csit/suites/sxp/cluster-routing/010_Route_Switching.robot b/csit/suites/sxp/cluster-routing/010_Route_Switching.robot index cbb4207e7f..c2717a2a96 100644 --- a/csit/suites/sxp/cluster-routing/010_Route_Switching.robot +++ b/csit/suites/sxp/cluster-routing/010_Route_Switching.robot @@ -70,10 +70,13 @@ Isolate SXP Controller [Documentation] Isolate the cluster leader node and perform check that virtual IP is routed to a new leader, ... afterwards unisolate old leader. @{running_members} = ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${running_members} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${controller_index} ${running_member} = Collections.Get From List ${running_members} 0 ${active_follower} = SxpClusterLib.Get Owner Controller ${running_member} BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${active_follower} ClusterManagement.Flush_Iptables_From_List_Or_All - BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Check_Cluster_Is_In_Sync + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${EMPTY} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} diff --git a/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot b/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot index edb515ece4..01e7e347ee 100644 --- a/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot +++ b/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot @@ -104,7 +104,8 @@ Clean Custom SXP Cluster Session Clean Custom SXP Cluster [Documentation] Disconnect SXP cluster topology ClusterManagement.Flush_Iptables_From_List_Or_All - BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Check_Cluster_Is_In_Sync + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${EMPTY} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} SxpLib.Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION} SxpLib.Delete Node ${CLUSTER_NODE_ID} session=${CONTROLLER_SESSION} @@ -118,6 +119,8 @@ Isolate SXP Controller [Arguments] ${controller_index} [Documentation] Isolate cluster node specified by ${controller_index} and find new owner @{running_members} = ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${running_members} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${controller_index} ${running_member} = Collections.Get From List ${running_members} 0 diff --git a/csit/suites/sxp/clustering/010_Connection_switchover.robot b/csit/suites/sxp/clustering/010_Connection_switchover.robot index 8aeac109ba..54b9017fee 100644 --- a/csit/suites/sxp/clustering/010_Connection_switchover.robot +++ b/csit/suites/sxp/clustering/010_Connection_switchover.robot @@ -24,9 +24,12 @@ Isolation of SXP noservice follower Test Isolate SXP Controller [Arguments] ${controller_index} [Documentation] Isolate one of cluster nodes and perform check that Device is still connected afterwards reverts isolation - ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + ${running_members} = ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${running_members} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Device is Connected ${DEVICE_NODE_ID} session=${DEVICE_SESSION} ClusterManagement.Flush_Iptables_From_List_Or_All - BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Check_Cluster_Is_In_Sync + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${EMPTY} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Device is Connected ${DEVICE_NODE_ID} session=${DEVICE_SESSION} diff --git a/csit/suites/sxp/clustering/030_RPC_functionality.robot b/csit/suites/sxp/clustering/030_RPC_functionality.robot index 111d56b07f..cd89d91b03 100644 --- a/csit/suites/sxp/clustering/030_RPC_functionality.robot +++ b/csit/suites/sxp/clustering/030_RPC_functionality.robot @@ -30,6 +30,8 @@ Isolate SXP Controller : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ SxpLib.Add Bindings ${i+1}0 ${i+1}0.${i+1}0.${i+1}0.${i+1}0/32 node=${INADDR_ANY} session=ClusterManagement__session_${owner_controller} @{running_members} = ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${running_members} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} ${running_member} = Collections.Get From List ${running_members} 0 ${owner_controller} = SxpClusterLib.Get Owner Controller ${running_member} @@ -37,7 +39,8 @@ Isolate SXP Controller : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ SxpLib.Delete Bindings ${i+1}0 ${i+1}0.${i+1}0.${i+1}0.${i+1}0/32 node=${INADDR_ANY} session=ClusterManagement__session_${owner_controller} ClusterManagement.Flush_Iptables_From_List_Or_All - BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Check_Cluster_Is_In_Sync + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${EMPTY} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 30 1 Check Bindings Does Not Exist ${owner_controller} Check Bindings Exist diff --git a/csit/suites/sxp/clustering/040_Bindings_consistency.robot b/csit/suites/sxp/clustering/040_Bindings_consistency.robot index 1766bac250..30d56c1653 100644 --- a/csit/suites/sxp/clustering/040_Bindings_consistency.robot +++ b/csit/suites/sxp/clustering/040_Bindings_consistency.robot @@ -67,6 +67,8 @@ Isolate SXP Controller [Arguments] ${controller_index} ${node} ${session}=ClusterManagement__session_${controller_index} [Documentation] Isolate one of cluster nodes and perform check that device is connected @{running_members} = ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${running_members} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} ${running_member} = Collections.Get From List ${running_members} 0 BuiltIn.Set Test Variable ${RUNNING_MEMBER} ${running_member} BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} @@ -76,7 +78,8 @@ UnIsolate SXP Controller [Arguments] ${controller_index} [Documentation] Un-Isolate one of cluster nodes and perform check that device is connected ClusterManagement.Flush_Iptables_From_List_Or_All - BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Check_Cluster_Is_In_Sync + BuiltIn.Wait_Until_Keyword_Succeeds 60 1 ClusterManagement.Verify_Members_Are_Ready member_index_list=${EMPTY} verify_cluster_sync=True verify_restconf=True + ... verify_system_status=False service_list=@{EMPTY} BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Device is Connected ${DEVICE_NODE_ID} session=${DEVICE_SESSION} Check Bindings -- 2.36.6