X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FSxpClusterLib.robot;h=755ba15e9b509046b7ea81a74c9d23a0c13619f3;hb=1eabc68b39f032c22dd6c342d5a43fbcdbe14679;hp=cfee3d14120710185bd88b0316de4607070a6619;hpb=d20f32d975fd341aec0b28af8c2792090d21b90a;p=integration%2Ftest.git diff --git a/csit/libraries/SxpClusterLib.robot b/csit/libraries/SxpClusterLib.robot index cfee3d1412..755ba15e9b 100644 --- a/csit/libraries/SxpClusterLib.robot +++ b/csit/libraries/SxpClusterLib.robot @@ -1,30 +1,27 @@ *** Settings *** Documentation Library containing Keywords used for SXP cluster testing Library RequestsLibrary -Library ./Sxp.py -Resource ./SxpLib.robot Resource ./ClusterManagement.robot Resource ./SetupUtils.robot -Resource ../variables/Variables.robot +Resource ./SxpLib.robot *** Variables *** @{SHARD_OPER_LIST} inventory topology default entity-ownership @{SHARD_CONF_LIST} inventory topology default +@{SXP_PACKAGE} org.opendaylight.sxp ${DEVICE_SESSION} device_1 +${CONTROLLER_SESSION} ClusterManagement__session_1 ${DEVICE_NODE_ID} 1.1.1.1 ${CLUSTER_NODE_ID} 2.2.2.2 ${SXP_LOG_LEVEL} INFO -@{SXP_PACKAGE} org.opendaylight.sxp +${VIRTUAL_IP} ${TOOLS_SYSTEM_2_IP} +${VIRTUAL_IP_MASK} 255.255.255.0 +${VIRTUAL_INTERFACE} eth0 +${MAC_ADDRESS_TABLE} &{EMPTY} *** Keywords *** Setup SXP Cluster Session [Documentation] Create sessions asociated with SXP cluster setup - BuiltIn.Wait Until Keyword Succeeds 120 10 SxpLib.Prepare SSH Keys On Karaf ${TOOLS_SYSTEM_IP} ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD} - ... ${TOOLS_SYSTEM_PROMPT} /tmp - SxpLib.Setup SXP Session ${DEVICE_SESSION} ${TOOLS_SYSTEM_IP} - : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ BuiltIn.Wait Until Keyword Succeeds 120 10 SxpLib.Prepare SSH Keys On Karaf ${ODL_SYSTEM_${i+1}_IP} - \ SxpLib.Setup SXP Session controller${i+1} ${ODL_SYSTEM_${i+1}_IP} ClusterManagement.ClusterManagement_Setup SetupUtils.Setup_Utils_For_Setup_And_Teardown SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All ${SXP_LOG_LEVEL} ${SXP_PACKAGE} @@ -32,8 +29,7 @@ Setup SXP Cluster Session Clean SXP Cluster Session [Documentation] Clean sessions asociated with SXP cluster setup ClusterManagement.Flush_Iptables_From_List_Or_All - : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${i+1} + ClusterManagement.Check_Cluster_Is_In_Sync SxpLib.Clean SXP Session SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All INFO ${SXP_PACKAGE} @@ -49,11 +45,11 @@ Setup SXP Cluster SxpLib.Add Node ${DEVICE_NODE_ID} ip=0.0.0.0 session=${DEVICE_SESSION} BuiltIn.Wait Until Keyword Succeeds 20 1 SxpLib.Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION} system=${TOOLS_SYSTEM_IP} ... ip=${EMPTY} - ${cluster_mode} Sxp.Get Opposing Mode ${peer_mode} + ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ SxpLib.Add Connection version4 ${peer_mode} ${ODL_SYSTEM_${i+1}_IP} 64999 ${DEVICE_NODE_ID} \ ... session=${DEVICE_SESSION} - ${controller_id} Get Any Controller + ${controller_id} = Get Any Controller SxpLib.Add Node ${CLUSTER_NODE_ID} ip=0.0.0.0 session=controller${controller_id} BuiltIn.Wait Until Keyword Succeeds 20 1 Check Cluster Node started ${CLUSTER_NODE_ID} SxpLib.Add Connection version4 ${cluster_mode} ${TOOLS_SYSTEM_IP} 64999 ${CLUSTER_NODE_ID} session=controller${controller_id} @@ -64,65 +60,59 @@ Clean SXP Cluster ClusterManagement.Flush_Iptables_From_List_Or_All : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ BuiltIn.Wait Until Keyword Succeeds 240 1 ClusterManagement.Sync_Status_Should_Be_True ${i+1} - ${controller_index} Get Active Controller + ${controller_index} = Get Owner Controller SxpLib.Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION} SxpLib.Delete Node ${CLUSTER_NODE_ID} session=controller${controller_index} Check Cluster Node started [Arguments] ${node} ${port}=64999 ${ip}=${EMPTY} [Documentation] Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes - ${started} BuiltIn.Set Variable ${False} + ${started} = BuiltIn.Set Variable ${False} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ ${rc} Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + \ ${rc} = Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} \ ... prompt=${ODL_SYSTEM_PROMPT} - \ ${started} BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started} + \ ${started} = BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started} BuiltIn.Should Be True ${started} Check Device is Connected [Arguments] ${node} ${version}=version4 ${port}=64999 ${session}=session [Documentation] Checks if SXP device is connected to at least one cluster node - ${is_connected} BuiltIn.Set Variable ${False} - ${resp} SxpLib.Get Connections node=${node} session=${session} + ${is_connected} = BuiltIn.Set Variable ${False} + ${resp} = SxpLib.Get Connections node=${node} session=${session} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ ${follower} Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP} + \ ${follower} = Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP} \ ... ${port} on - \ ${is_connected} BuiltIn.Run Keyword If ${follower} BuiltIn.Set Variable ${True} + \ ${is_connected} = BuiltIn.Run Keyword If ${follower} BuiltIn.Set Variable ${True} \ ... ELSE BuiltIn.Set Variable ${is_connected} BuiltIn.Should Be True ${is_connected} Check Cluster is Connected [Arguments] ${node} ${version}=version4 ${port}=64999 ${mode}=speaker ${session}=session [Documentation] Checks if SXP device is connected to at least one cluster node - ${resp} SxpLib.Get Connections node=${node} session=${session} + ${resp} = SxpLib.Get Connections node=${node} session=${session} SxpLib.Should Contain Connection ${resp} ${TOOLS_SYSTEM_IP} ${port} ${mode} ${version} -Get Active Controller - [Documentation] Find cluster controller that is marked as leader for SXP service in cluster - @{votes} BuiltIn.Create List - : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} - \ ${resp} RequestsLibrary.Get Request controller${i+1} /restconf/operational/entity-owners:entity-owners - \ BuiltIn.Continue For Loop If ${resp.status_code} != 200 - \ ${controller} Sxp.Get Active Controller From Json ${resp.content} SxpControllerInstance - \ Collections.Append To List ${votes} ${controller} - ${length} BuiltIn.Get Length ${votes} - BuiltIn.Should Not Be Equal As Integers ${length} 0 - ${active_controller} BuiltIn.Evaluate collections.Counter(${votes}).most_common(1)[0][0] collections - [Return] ${active_controller} - -Get Inactive Controller - [Documentation] Find cluster controller that is not marked as leader for SXP service in cluster - ${active_controller} Get Active Controller - ${controller} BuiltIn.Evaluate random.choice( filter( lambda i: i!=${active_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random +Get Owner Controller + [Arguments] ${running_member}=1 + [Documentation] Find cluster controller that is marked as cluster owner by requesting ownership data from ${running_member} node of the cluster + ${owner} ${candidates} = BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Get_Owner_And_Successors_For_Device org.opendaylight.sxp.controller.boot.SxpControllerInstance + ... Sxp ${running_member} + [Return] ${owner} + +Get Not Owner Controller + [Documentation] Find cluster controller that is not marked as owner for SXP service in cluster + ${owner_controller} = Get Owner Controller + ${controller} = BuiltIn.Evaluate random.choice( filter( lambda i: i!=${owner_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random [Return] ${controller} Get Any Controller [Documentation] Get any controller from cluster range - ${follower} BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random + ${follower} = BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random [Return] ${follower} Map Followers To Mac Addresses [Documentation] Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS - ${mac_addresses} BuiltIn.Create dictionary + ${mac_addresses} = BuiltIn.Create dictionary : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ ${mac_address} Find Mac Address Of Ip Address ${ODL_SYSTEM_${i+1}_IP} \ Collections.Set To Dictionary ${mac_addresses} ${ODL_SYSTEM_${i+1}_IP} ${mac_address} @@ -132,28 +122,42 @@ Map Followers To Mac Addresses Find Mac Address Of Ip Address [Arguments] ${ip} [Documentation] Finds out MAC-ADDRESS of specified IP by pinging it from TOOLS_SYSTEM machine - ${mac_address} Utils.Run Command On Remote System ${TOOLS_SYSTEM_IP} ping -c 1 -W 1 ${ip} >/dev/null && arp -n | grep ${ip} | awk '{print $3}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD} + ${mac_address} = Utils.Run Command On Remote System ${TOOLS_SYSTEM_IP} ping -c 1 -W 1 ${ip} >/dev/null && arp -n | grep ${ip} | awk '{print $3}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD} [Return] ${mac_address} Ip Addres Should Not Be Routed To Follower [Arguments] ${mac_addresses} ${ip_address} ${follower_index} [Documentation] Verify that IP-ADDRESS is not routed to follower specified by ID - ${mac_address_assigned} Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} - ${mac_address_resolved} Find Mac Address Of Ip Address ${ip_address} + ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} + ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address} BuiltIn.Should Not Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved} Ip Addres Should Be Routed To Follower [Arguments] ${mac_addresses} ${ip_address} ${follower_index} [Documentation] Verify that IP-ADDRESS is routed to follower specified by ID - ${mac_address_assigned} Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} - ${mac_address_resolved} Find Mac Address Of Ip Address ${ip_address} + ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP} + ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address} BuiltIn.Should Not Be Empty ${mac_address_resolved} BuiltIn.Should Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved} Shutdown Tools Node [Arguments] ${ip_address}=${TOOLS_SYSTEM_2_IP} ${user}=${TOOLS_SYSTEM_USER} ${passwd}=${TOOLS_SYSTEM_PASSWORD} [Documentation] Shutdown Tools node to avoid conflict in resolving virtual ip that is overlaping that node. - ${rc} OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address} - ${stdout} BuiltIn.Run Keyword And Return If ${rc} == 0 Utils.Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user} + ${rc} = OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address} + ${stdout} = BuiltIn.Run Keyword And Return If ${rc} == 0 Utils.Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user} ... ${passwd} BuiltIn.Log ${stdout} + +Create Virtual Interface + [Documentation] Create virtual interface on all of the cluster nodes + : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} + \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo modprobe dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo ip link set name ${VIRTUAL_INTERFACE} dev dummy0 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + \ Utils.Run Command On Remote System And Log ${ODL_SYSTEM_${i+1}_IP} sudo ip link show ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + +Delete Virtual Interface + [Documentation] Create virtual interface on all of the cluster nodes + : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} + \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo ip link delete ${VIRTUAL_INTERFACE} type dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo rmmod dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD} + \ Utils.Run Command On Remote System And Log ${ODL_SYSTEM_${i+1}_IP} sudo ip link show ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}