From d63ac515c3c5c4d6a6c45f6d0272c90fbb18e4ad Mon Sep 17 00:00:00 2001 From: Ivan Hrasko Date: Fri, 17 Aug 2018 14:35:38 +0200 Subject: [PATCH] SXP: Format cluster-routing suite Change-Id: I3e839826e200f5f4c022e1909e7969fee24d0c85 Signed-off-by: Ivan Hrasko --- csit/libraries/SxpClusterLib.robot | 54 ++++---- .../cluster-routing/010_Route_Switching.robot | 66 +++++----- .../020_Sxp_Node_Switching.robot | 122 +++++++++--------- 3 files changed, 121 insertions(+), 121 deletions(-) diff --git a/csit/libraries/SxpClusterLib.robot b/csit/libraries/SxpClusterLib.robot index cfee3d1412..e5b2dfaa1f 100644 --- a/csit/libraries/SxpClusterLib.robot +++ b/csit/libraries/SxpClusterLib.robot @@ -49,11 +49,11 @@ Setup SXP Cluster SxpLib.Add Node ${DEVICE_NODE_ID} ip=0.0.0.0 session=${DEVICE_SESSION} BuiltIn.Wait Until Keyword Succeeds 20 1 SxpLib.Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION} system=${TOOLS_SYSTEM_IP} ... ip=${EMPTY} - ${cluster_mode} Sxp.Get Opposing Mode ${peer_mode} + ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ SxpLib.Add Connection version4 ${peer_mode} ${ODL_SYSTEM_${i+1}_IP} 64999 ${DEVICE_NODE_ID} \ ... session=${DEVICE_SESSION} - ${controller_id} Get Any Controller + ${controller_id} = Get Any Controller SxpLib.Add Node ${CLUSTER_NODE_ID} ip=0.0.0.0 session=controller${controller_id} BuiltIn.Wait Until Keyword Succeeds 20 1 Check Cluster Node started ${CLUSTER_NODE_ID} SxpLib.Add Connection version4 ${cluster_mode} ${TOOLS_SYSTEM_IP} 64999 ${CLUSTER_NODE_ID} session=controller${controller_id} @@ -64,65 +64,65 @@ Clean SXP Cluster ClusterManagement.Flush_Iptables_From_List_Or_All : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${i+1} - ${controller_index} Get Active Controller + ${controller_index} = Get Active Controller SxpLib.Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION} SxpLib.Delete Node ${CLUSTER_NODE_ID} session=controller${controller_index} Check Cluster Node started [Arguments] ${node} ${port}=64999 ${ip}=${EMPTY} [Documentation] Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes - ${started} BuiltIn.Set Variable ${False} + ${started} = BuiltIn.Set Variable ${False} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ ${rc} Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + \ ${rc} = Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} \ ... prompt=${ODL_SYSTEM_PROMPT} - \ ${started} BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started} + \ ${started} = BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started} BuiltIn.Should Be True ${started} Check Device is Connected [Arguments] ${node} ${version}=version4 ${port}=64999 ${session}=session [Documentation] Checks if SXP device is connected to at least one cluster node - ${is_connected} BuiltIn.Set Variable ${False} - ${resp} SxpLib.Get Connections node=${node} session=${session} + ${is_connected} = BuiltIn.Set Variable ${False} + ${resp} = SxpLib.Get Connections node=${node} session=${session} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ ${follower} Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP} + \ ${follower} = Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP} \ ... ${port} on - \ ${is_connected} BuiltIn.Run Keyword If ${follower} BuiltIn.Set Variable ${True} + \ ${is_connected} = BuiltIn.Run Keyword If ${follower} BuiltIn.Set Variable ${True} \ ... ELSE BuiltIn.Set Variable ${is_connected} BuiltIn.Should Be True ${is_connected} Check Cluster is Connected [Arguments] ${node} ${version}=version4 ${port}=64999 ${mode}=speaker ${session}=session [Documentation] Checks if SXP device is connected to at least one cluster node - ${resp} SxpLib.Get Connections node=${node} session=${session} + ${resp} = SxpLib.Get Connections node=${node} session=${session} SxpLib.Should Contain Connection ${resp} ${TOOLS_SYSTEM_IP} ${port} ${mode} ${version} Get Active Controller [Documentation] Find cluster controller that is marked as leader for SXP service in cluster - @{votes} BuiltIn.Create List + @{votes} = BuiltIn.Create List : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ ${resp} RequestsLibrary.Get Request controller${i+1} /restconf/operational/entity-owners:entity-owners + \ ${resp} = RequestsLibrary.Get Request controller${i+1} /restconf/operational/entity-owners:entity-owners \ BuiltIn.Continue For Loop If ${resp.status_code} != 200 - \ ${controller} Sxp.Get Active Controller From Json ${resp.content} SxpControllerInstance + \ ${controller} = Sxp.Get Active Controller From Json ${resp.content} SxpControllerInstance \ Collections.Append To List ${votes} ${controller} - ${length} BuiltIn.Get Length ${votes} + ${length} = BuiltIn.Get Length ${votes} BuiltIn.Should Not Be Equal As Integers ${length} 0 - ${active_controller} BuiltIn.Evaluate collections.Counter(${votes}).most_common(1)[0][0] collections + ${active_controller} = BuiltIn.Evaluate collections.Counter(${votes}).most_common(1)[0][0] collections [Return] ${active_controller} Get Inactive Controller [Documentation] Find cluster controller that is not marked as leader for SXP service in cluster - ${active_controller} Get Active Controller - ${controller} BuiltIn.Evaluate random.choice( filter( lambda i: i!=${active_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random + ${active_controller} = Get Active Controller + ${controller} = BuiltIn.Evaluate random.choice( filter( lambda i: i!=${active_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random [Return] ${controller} Get Any Controller [Documentation] Get any controller from cluster range - ${follower} BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random + ${follower} = BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random [Return] ${follower} Map Followers To Mac Addresses [Documentation] Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS - ${mac_addresses} BuiltIn.Create dictionary + ${mac_addresses} = BuiltIn.Create dictionary : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ ${mac_address} Find Mac Address Of Ip Address ${ODL_SYSTEM_${i+1}_IP} \ Collections.Set To Dictionary ${mac_addresses} ${ODL_SYSTEM_${i+1}_IP} ${mac_address} @@ -132,28 +132,28 @@ Map Followers To Mac Addresses Find Mac Address Of Ip Address [Arguments] ${ip} [Documentation] Finds out MAC-ADDRESS of specified IP by pinging it from TOOLS_SYSTEM machine - ${mac_address} Utils.Run Command On Remote System ${TOOLS_SYSTEM_IP} ping -c 1 -W 1 ${ip} >/dev/null && arp -n | grep ${ip} | awk '{print $3}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD} + ${mac_address} = Utils.Run Command On Remote System ${TOOLS_SYSTEM_IP} ping -c 1 -W 1 ${ip} >/dev/null && arp -n | grep ${ip} | awk '{print $3}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD} [Return] ${mac_address} Ip Addres Should Not Be Routed To Follower [Arguments] ${mac_addresses} ${ip_address} ${follower_index} [Documentation] Verify that IP-ADDRESS is not routed to follower specified by ID - ${mac_address_assigned} Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} - ${mac_address_resolved} Find Mac Address Of Ip Address ${ip_address} + ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} + ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address} BuiltIn.Should Not Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved} Ip Addres Should Be Routed To Follower [Arguments] ${mac_addresses} ${ip_address} ${follower_index} [Documentation] Verify that IP-ADDRESS is routed to follower specified by ID - ${mac_address_assigned} Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} - ${mac_address_resolved} Find Mac Address Of Ip Address ${ip_address} + ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} + ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address} BuiltIn.Should Not Be Empty ${mac_address_resolved} BuiltIn.Should Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved} Shutdown Tools Node [Arguments] ${ip_address}=${TOOLS_SYSTEM_2_IP} ${user}=${TOOLS_SYSTEM_USER} ${passwd}=${TOOLS_SYSTEM_PASSWORD} [Documentation] Shutdown Tools node to avoid conflict in resolving virtual ip that is overlaping that node. - ${rc} OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address} - ${stdout} BuiltIn.Run Keyword And Return If ${rc} == 0 Utils.Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user} + ${rc} = OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address} + ${stdout} = BuiltIn.Run Keyword And Return If ${rc} == 0 Utils.Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user} ... ${passwd} BuiltIn.Log ${stdout} diff --git a/csit/suites/sxp/cluster-routing/010_Route_Switching.robot b/csit/suites/sxp/cluster-routing/010_Route_Switching.robot index 878eca48dd..74abb6fe9e 100644 --- a/csit/suites/sxp/cluster-routing/010_Route_Switching.robot +++ b/csit/suites/sxp/cluster-routing/010_Route_Switching.robot @@ -19,68 +19,68 @@ ${VIRTUAL_IP_MASK_1} 255.255.255.0 *** Test Cases *** Route Definition Test [Documentation] Test Route update mechanism without cluster node isolation - Check Shards Status - ${active_controller} Get Active Controller - Wait Until Keyword Succeeds 240 1 Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} + SxpClusterLib.Check Shards Status + ${active_controller} = SxpClusterLib.Get Active Controller + BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} Add Route Definition To Cluster ${VIRTUAL_IP_1} ${VIRTUAL_IP_MASK_1} ${VIRTUAL_INTERFACE_1} ${active_controller} - Wait Until Keyword Succeeds 240 1 Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} - Clean Routing Configuration To Controller controller${active_controller} - Wait Until Keyword Succeeds 240 1 Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} + BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} + SxpLib.Clean Routing Configuration To Controller controller${active_controller} + BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} Put Route Definition To Cluster ${VIRTUAL_IP_1} ${VIRTUAL_IP_MASK_1} ${VIRTUAL_INTERFACE_1} ${active_controller} - Wait Until Keyword Succeeds 240 1 Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} + BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_controller} Isolation of SXP service follower Test [Documentation] Test Route update mechanism during Cluster isolation, ... after each isolation virtual ip should be pre-routed to new leader - Check Shards Status - ${any_controller} Get Any Controller + SxpClusterLib.Check Shards Status + ${any_controller} = SxpClusterLib.Get Any Controller Add Route Definition To Cluster ${VIRTUAL_IP_1} ${VIRTUAL_IP_MASK_1} ${VIRTUAL_INTERFACE_1} ${any_controller} : FOR ${i} IN RANGE 0 ${SAMPLES} - \ ${controller_index} Get Active Controller + \ ${controller_index} = SxpClusterLib.Get Active Controller \ Isolate SXP Controller ${controller_index} *** Keywords *** Put Route Definition To Cluster [Arguments] ${virtual_ip} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} ${follower} [Documentation] Put Route definition to DS replacing all present - ${route} Route Definition Xml ${virtual_ip} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} - ${routes} Route Definitions Xml ${route} - Put Routing Configuration To Controller ${routes} controller${follower} + ${route} = Sxp.Route Definition Xml ${virtual_ip} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} + ${routes} = Sxp.Route Definitions Xml ${route} + SxpLib.Put Routing Configuration To Controller ${routes} controller${follower} Add Route Definition To Cluster [Arguments] ${VIRTUAL_IP} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} ${follower} [Documentation] Add Route definition to DS - ${old_routes} Get Routing Configuration From Controller controller${follower} - ${route} Route Definition Xml ${VIRTUAL_IP} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} - ${routes} Route Definitions Xml ${route} ${old_routes} - Put Routing Configuration To Controller ${routes} controller${follower} + ${old_routes} = SxpLib.Get Routing Configuration From Controller controller${follower} + ${route} = Sxp.Route Definition Xml ${VIRTUAL_IP} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} + ${routes} = Sxp.Route Definitions Xml ${route} ${old_routes} + SxpLib.Put Routing Configuration To Controller ${routes} controller${follower} Custom Clean SXP Cluster [Documentation] Cleans up Route definitions - ${follower} Get Active Controller - Clean Routing Configuration To Controller controller${follower} + ${follower} = SxpClusterLib.Get Active Controller + SxpLib.Clean Routing Configuration To Controller controller${follower} Setup Custom SXP Cluster Session [Documentation] Prepare topology for testing, creates sessions and generate Route definitions based on Cluster nodes ip - Shutdown Tools Node - Setup SXP Cluster Session - ${mac_addresses} Map Followers To Mac Addresses - Set Suite Variable ${MAC_ADDRESS_TABLE} ${mac_addresses} + SxpClusterLib.Shutdown Tools Node + SxpClusterLib.Setup SXP Cluster Session + ${mac_addresses} = SxpClusterLib.Map Followers To Mac Addresses + BuiltIn.Set Suite Variable ${MAC_ADDRESS_TABLE} ${mac_addresses} Clean Custom SXP Cluster Session [Documentation] Cleans up resources generated by test - ${controller_index} Get Active Controller - Clean Routing Configuration To Controller controller${controller_index} - Clean SXP Cluster Session + ${controller_index} = SxpClusterLib.Get Active Controller + SxpLib.Clean Routing Configuration To Controller controller${controller_index} + SxpClusterLib.Clean SXP Cluster Session Isolate SXP Controller [Arguments] ${controller_index} [Documentation] Isolate one of cluster nodes and perform check that virtual ip is routed to another cluster node, ... afterwards unisolate old leader. - Isolate_Member_From_List_Or_All ${controller_index} - Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_False ${controller_index} - Wait Until Keyword Succeeds 240 1 Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${controller_index} - ${active_follower} Get Active Controller - Wait Until Keyword Succeeds 240 1 Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_follower} - Flush_Iptables_From_List_Or_All - Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_True ${controller_index} + ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 120 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 120 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${controller_index} + ${active_follower} = SxpClusterLib.Get Active Controller + BuiltIn.Wait Until Keyword Succeeds 120 1 SxpClusterLib.Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP_1} ${active_follower} + ClusterManagement.Flush_Iptables_From_List_Or_All + BuiltIn.Wait Until Keyword Succeeds 120 1 ClusterManagement.Sync_Status_Should_Be_True ${controller_index} diff --git a/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot b/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot index def466624b..3d3ff89816 100644 --- a/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot +++ b/csit/suites/sxp/cluster-routing/020_Sxp_Node_Switching.robot @@ -20,120 +20,120 @@ ${VIRTUAL_IP_MASK} 255.255.255.0 *** Test Cases *** Isolation of SXP service follower Test [Documentation] Test SXP connection switchover only if Controller with SCS is isolated - Check Shards Status + SxpClusterLib.Check Shards Status Setup Custom SXP Cluster ${VIRTUAL_IP} listener : FOR ${i} IN RANGE 0 ${SAMPLES} - \ ${controller_index} Get Active Controller + \ ${controller_index} = SxpClusterLib.Get Active Controller \ Isolate SXP Controller ${controller_index} listener Isolation of SXP service follower Test Listener Part [Documentation] Test SXP binding propagation only if Controller with SCS is isolated - Check Shards Status - ${controller_index} Get Active Controller + SxpClusterLib.Check Shards Status + ${controller_index} = SxpClusterLib.Get Active Controller Setup Custom SXP Cluster ${VIRTUAL_IP} listener Setup SXP Cluster Bindings ${CLUSTER_NODE_ID} controller${controller_index} : FOR ${i} IN RANGE 0 ${SAMPLES} - \ ${controller_index} Get Active Controller + \ ${controller_index} = SxpClusterLib.Get Active Controller \ Isolate SXP Controller With Bindings ${controller_index} ${DEVICE_NODE_ID} listener ${DEVICE_SESSION} Isolation of SXP service follower Test Speaker Part [Documentation] Test SXP binding propagation only if Controller with SCS is isolated, ... the same case as above but with initiator of connection between nodes in oposite mode - Check Shards Status + SxpClusterLib.Check Shards Status Setup Custom SXP Cluster ${VIRTUAL_IP} speaker Setup SXP Cluster Bindings ${DEVICE_NODE_ID} ${DEVICE_SESSION} : FOR ${i} IN RANGE 0 ${SAMPLES} - \ ${controller_index} Get Active Controller + \ ${controller_index} = SxpClusterLib.Get Active Controller \ Isolate SXP Controller With Bindings ${controller_index} ${CLUSTER_NODE_ID} speaker *** Keywords *** Setup Custom SXP Cluster Session [Documentation] Prepare topology for testing, creates sessions and generate Route definitions based on Cluster nodes ip - Shutdown Tools Node - Setup SXP Cluster Session - ${controller_index} Get Active Controller - ${mac_addresses} Map Followers To Mac Addresses - Set Suite Variable ${MAC_ADDRESS_TABLE} ${mac_addresses} - ${route} Route Definition Xml ${VIRTUAL_IP} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} - ${routes} Route Definitions Xml ${route} - Put Routing Configuration To Controller ${routes} controller${controller_index} + SxpClusterLib.Shutdown Tools Node + SxpClusterLib.Setup SXP Cluster Session + ${controller_index} = SxpClusterLib.Get Active Controller + ${mac_addresses} = SxpClusterLib.Map Followers To Mac Addresses + BuiltIn.Set Suite Variable ${MAC_ADDRESS_TABLE} ${mac_addresses} + ${route} = Sxp.Route Definition Xml ${VIRTUAL_IP} ${VIRTUAL_IP_MASK} ${VIRTUAL_INTERFACE} + ${routes} = Sxp.Route Definitions Xml ${route} + SxpLib.Put Routing Configuration To Controller ${routes} controller${controller_index} Clean Custom SXP Cluster Session [Documentation] Cleans up resources generated by test - ${controller_index} Get Active Controller - Clean Routing Configuration To Controller controller${controller_index} - Clean SXP Cluster Session + ${controller_index} = SxpClusterLib.Get Active Controller + SxpLib.Clean Routing Configuration To Controller controller${controller_index} + SxpClusterLib.Clean SXP Cluster Session Setup Custom SXP Cluster [Arguments] ${peer_address} ${peer_mode} [Documentation] Setup and connect SXP cluster topology - Add Node ${DEVICE_NODE_ID} ip=0.0.0.0 session=${DEVICE_SESSION} - Wait Until Keyword Succeeds 20 1 Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION} system=${TOOLS_SYSTEM_IP} + SxpLib.Add Node ${DEVICE_NODE_ID} ip=0.0.0.0 session=${DEVICE_SESSION} + BuiltIn.Wait Until Keyword Succeeds 20 1 SxpLib.Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION} system=${TOOLS_SYSTEM_IP} ... ip=${EMPTY} - ${cluster_mode} Get Opposing Mode ${peer_mode} - Add Connection version4 ${peer_mode} ${peer_address} 64999 ${DEVICE_NODE_ID} session=${DEVICE_SESSION} - ${controller_id} Get Active Controller - Add Node ${CLUSTER_NODE_ID} ip=${peer_address} session=controller${controller_id} - Wait Until Keyword Succeeds 20 1 Check Cluster Node started ${CLUSTER_NODE_ID} - Add Connection version4 ${cluster_mode} ${TOOLS_SYSTEM_IP} 64999 ${CLUSTER_NODE_ID} session=controller${controller_id} - Wait Until Keyword Succeeds 120 1 Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${controller_id} + ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode} + SxpLib.Add Connection version4 ${peer_mode} ${peer_address} 64999 ${DEVICE_NODE_ID} session=${DEVICE_SESSION} + ${controller_id} = SxpClusterLib.Get Active Controller + SxpLib.Add Node ${CLUSTER_NODE_ID} ip=${peer_address} session=controller${controller_id} + BuiltIn.Wait Until Keyword Succeeds 20 1 SxpClusterLib.Check Cluster Node started ${CLUSTER_NODE_ID} + SxpLib.Add Connection version4 ${cluster_mode} ${TOOLS_SYSTEM_IP} 64999 ${CLUSTER_NODE_ID} session=controller${controller_id} + BuiltIn.Wait Until Keyword Succeeds 120 1 SxpClusterLib.Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${controller_id} Setup SXP Cluster Bindings [Arguments] ${node} ${session} [Documentation] Setup initial bindings to SXP device : FOR ${i} IN RANGE 1 ${BINDINGS} - \ Add Bindings ${i}0 ${i}.${i}.${i}.${i}/32 node=${node} session=${session} + \ SxpLib.Add Bindings ${i}0 ${i}.${i}.${i}.${i}/32 node=${node} session=${session} Isolate SXP Controller [Arguments] ${controller_index} ${peer_mode} [Documentation] Isolate one of cluster nodes and perform check that Device is still connected then revert isolation (and check connection again). - ${cluster_mode} Get Opposing Mode ${peer_mode} - Isolate_Member_From_List_Or_All ${controller_index} - Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_False ${controller_index} - Wait Until Keyword Succeeds 240 1 Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${controller_index} - ${active_follower} Get Active Controller - Wait Until Keyword Succeeds 240 1 Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${active_follower} - Wait Until Keyword Succeeds 60 1 Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} - Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} + ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode} + ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${controller_index} + ${active_follower} = SxpClusterLib.Get Active Controller + BuiltIn.Wait Until Keyword Succeeds 240 1 SXpClusterLib.Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${active_follower} + BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} + BuiltIn.Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} ... session=${DEVICE_SESSION} - Flush_Iptables_From_List_Or_All - Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_True ${controller_index} - Wait Until Keyword Succeeds 60 1 Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} - Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} + ClusterManagement.Flush_Iptables_From_List_Or_All + BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} + BuiltIn.Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} ... session=${DEVICE_SESSION} Isolate SXP Controller With Bindings [Arguments] ${controller_index} ${node} ${peer_mode} ${session}=${EMPTY} [Documentation] Isolate one of cluster nodes and perform check that bindings were propagated then revert isolation (and check connection again). - ${find_session} Set Variable If '${session}' == '${EMPTY}' ${True} ${False} - ${cluster_mode} Get Opposing Mode ${peer_mode} - ${session} Set Variable If ${find_session} controller${controller_index} ${session} - Isolate_Member_From_List_Or_All ${controller_index} - Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_False ${controller_index} - Wait Until Keyword Succeeds 240 1 Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${controller_index} - ${active_follower} Get Active Controller - Wait Until Keyword Succeeds 240 1 Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${active_follower} - Wait Until Keyword Succeeds 60 1 Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} - Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} + ${find_session} = BuiltIn.Set Variable If '${session}' == '${EMPTY}' ${True} ${False} + ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode} + ${session} = BuiltIn.Set Variable If ${find_session} controller${controller_index} ${session} + ClusterManagement.Isolate_Member_From_List_Or_All ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_False ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 240 1 SxpClusterLib.Ip Addres Should Not Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${controller_index} + ${active_follower} = SxpClusterLib.Get Active Controller + BuiltIn.Wait Until Keyword Succeeds 240 1 SXpClusterLib.Ip Addres Should Be Routed To Follower ${MAC_ADDRESS_TABLE} ${VIRTUAL_IP} ${active_follower} + BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} + BuiltIn.Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} ... session=${DEVICE_SESSION} - ${session} Set Variable If ${find_session} controller${active_follower} ${session} - Wait Until Keyword Succeeds 30 1 Check Bindings ${node} ${session} - Flush_Iptables_From_List_Or_All - Wait Until Keyword Succeeds 240 1 Sync_Status_Should_Be_True ${controller_index} - Wait Until Keyword Succeeds 60 1 Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} - Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} + ${session} = BuiltIn.Set Variable If ${find_session} controller${active_follower} ${session} + BuiltIn.Wait Until Keyword Succeeds 30 1 Check Bindings ${node} ${session} + ClusterManagement.Flush_Iptables_From_List_Or_All + BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${controller_index} + BuiltIn.Wait Until Keyword Succeeds 60 1 SxpClusterLib.Check Cluster is Connected ${CLUSTER_NODE_ID} mode=${cluster_mode} session=controller${active_follower} + BuiltIn.Wait Until Keyword Succeeds 60 1 Check Device is Connected ${DEVICE_NODE_ID} ${VIRTUAL_IP} ${peer_mode} ... session=${DEVICE_SESSION} - Wait Until Keyword Succeeds 30 1 Check Bindings ${node} ${session} + BuiltIn.Wait Until Keyword Succeeds 30 1 Check Bindings ${node} ${session} Check Device is Connected [Arguments] ${node} ${remote_ip} ${mode}=any ${version}=version4 ${port}=64999 ${session}=session [Documentation] Checks if SXP device is connected to at least one cluster node - ${resp} Get Connections node=${node} session=${session} - Should Contain Connection ${resp} ${remote_ip} ${port} ${mode} ${version} + ${resp} = SxpLib.Get Connections node=${node} session=${session} + SxpLib.Should Contain Connection ${resp} ${remote_ip} ${port} ${mode} ${version} Check Bindings [Arguments] ${node} ${session} [Documentation] Checks that bindings were propagated to Peer - ${resp} Get Bindings node=${node} session=${session} + ${resp} SxpLib.Get Bindings node=${node} session=${session} : FOR ${i} IN RANGE 1 ${BINDINGS} - \ Should Contain Binding ${resp} ${i}0 ${i}.${i}.${i}.${i}/32 + \ SxpLib.Should Contain Binding ${resp} ${i}0 ${i}.${i}.${i}.${i}/32 -- 2.36.6