2 Documentation Library containing Keywords used for SXP cluster testing
3 Library RequestsLibrary
4 Resource ./ClusterManagement.robot
5 Resource ./SetupUtils.robot
6 Resource ./SxpLib.robot
9 @{SHARD_OPER_LIST} inventory topology default entity-ownership
10 @{SHARD_CONF_LIST} inventory topology default
11 @{SXP_PACKAGE} org.opendaylight.sxp
12 ${DEVICE_SESSION} device_1
13 ${CONTROLLER_SESSION} ClusterManagement__session_1
15 ${VIRTUAL_IP} ${TOOLS_SYSTEM_2_IP}
16 ${VIRTUAL_IP_MASK} 255.255.255.0
17 ${VIRTUAL_INTERFACE} dummy0
18 ${MAC_ADDRESS_TABLE} &{EMPTY}
19 ${DEVICE_NODE_ID} ${TOOLS_SYSTEM_IP}
20 ${CLUSTER_NODE_ID} ${TOOLS_SYSTEM_2_IP}
24 Setup SXP Cluster Session
25 [Documentation] Create sessions asociated with SXP cluster setup
26 ClusterManagement.ClusterManagement_Setup
27 SetupUtils.Setup_Utils_For_Setup_And_Teardown
28 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All ${SXP_LOG_LEVEL} ${SXP_PACKAGE}
31 [Documentation] Create session on the SXP device
32 RequestsLibrary.Create Session ${DEVICE_SESSION} url=http://${DEVICE_NODE_ID}:${RESTCONFPORT} auth=${AUTH} timeout=${DEFAULT_TIMEOUT_HTTP} max_retries=0
34 Setup SXP Cluster Session With Device
35 [Documentation] Create sessions asociated with SXP cluster setup and one SXP device
36 Setup SXP Cluster Session
39 Clean SXP Cluster Session
40 [Documentation] Clean sessions asociated with SXP cluster setup
41 ClusterManagement.Flush_Iptables_From_List_Or_All
42 RequestsLibrary.Delete All Sessions
43 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All INFO ${SXP_PACKAGE}
46 [Documentation] Check Status for all shards in SXP application.
47 ClusterManagement.Check_Cluster_Is_In_Sync
48 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_OPER_LIST} shard_type=operational
49 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_CONF_LIST} shard_type=config
52 [Arguments] ${peer_mode}=listener
53 [Documentation] Setup and connect SXP cluster topology
54 SxpLib.Add Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
55 BuiltIn.Wait Until Keyword Succeeds 20x 10s SxpLib.Check Node Started ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
56 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
57 \ SxpLib.Add Connection version4 ${peer_mode} ${ODL_SYSTEM_${i+1}_IP} 64999 node=${DEVICE_NODE_ID}
58 \ ... session=${DEVICE_SESSION}
59 ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode}
60 SxpLib.Add Node ${INADDR_ANY} session=${CONTROLLER_SESSION}
61 BuiltIn.Wait Until Keyword Succeeds 20x 10s Check Cluster Node Started ${INADDR_ANY} ip=${EMPTY}
62 SxpLib.Add Connection version4 ${cluster_mode} ${DEVICE_NODE_ID} 64999 ${INADDR_ANY} session=${CONTROLLER_SESSION}
63 BuiltIn.Wait Until Keyword Succeeds 48x 10s Check Device is Connected ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
66 [Documentation] Disconnect SXP cluster topology
67 SxpLib.Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
68 BuiltIn.Wait Until Keyword Succeeds 12x 10s SxpLib.Check Node Stopped ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
69 BuiltIn.Wait Until Keyword Succeeds 3x 10s SxpLib.Delete Node ${INADDR_ANY} session=${CONTROLLER_SESSION}
70 BuiltIn.Wait Until Keyword Succeeds 12x 10s SxpClusterLib.Check Cluster Node Stopped ${INADDR_ANY} ip=${EMPTY}
72 Check Cluster Node Started
73 [Arguments] ${node} ${port}=64999 ${ip}=${node}
74 [Documentation] Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes
75 ${resp} = RequestsLibrary.Get Request ${CONTROLLER_SESSION} /restconf/operational/network-topology:network-topology/topology/sxp/node/${node}/
76 BuiltIn.Should Be Equal As Strings ${resp.status_code} 200
77 ${started} = BuiltIn.Set Variable ${False}
78 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
79 \ ${rc} = Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
80 \ ... prompt=${ODL_SYSTEM_PROMPT}
81 \ ${started} = BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started}
82 BuiltIn.Should Be True ${started}
84 Check Cluster Node Stopped
85 [Arguments] ${node} ${port}=64999 ${ip}=${node}
86 [Documentation] Verify that SxpNode has data removed from Operational datastore and Node is stopped
87 ${resp} = RequestsLibrary.Get Request ${CONTROLLER_SESSION} /restconf/operational/network-topology:network-topology/topology/sxp/node/${node}/
88 BuiltIn.Should Be Equal As Strings ${resp.status_code} 404
89 ${stopped} = BuiltIn.Set Variable ${False}
90 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
91 \ ${rc} = Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1 ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
92 \ ... prompt=${ODL_SYSTEM_PROMPT}
93 \ ${stopped} = BuiltIn.Set Variable If '${rc}' == '1' ${True} ${stopped}
94 BuiltIn.Should Be True ${stopped}
96 Check Device is Connected
97 [Arguments] ${node} ${version}=version4 ${port}=64999 ${session}=session
98 [Documentation] Checks if SXP device is connected to the cluster. It means it has connection in state "on" with one of the cluster members.
99 ${resp} = SxpLib.Get Connections node=${node} session=${session}
100 ${is_connected} = BuiltIn.Set Variable ${False}
101 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
102 \ ${is_connected} = Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP}
104 \ BuiltIn.Exit For Loop If ${is_connected}
105 BuiltIn.Should Be True ${is_connected}
107 Check Cluster is Connected
108 [Arguments] ${node} ${version}=version4 ${port}=64999 ${mode}=speaker ${session}=session
109 [Documentation] Get SXP connections of cluster ${node} and verify that they contain a connection to the device ${DEVICE_NODE_ID} in state "on"
110 ${resp} = SxpLib.Get Connections node=${node} session=${session}
111 SxpLib.Should Contain Connection ${resp} ${DEVICE_NODE_ID} ${port} ${mode} ${version} on
114 [Arguments] ${running_member}=1
115 [Documentation] Find cluster controller that is marked as cluster owner by requesting ownership data from ${running_member} node of the cluster
116 ${owner} ${candidates} = BuiltIn.Wait Until Keyword Succeeds 5x 2s ClusterManagement.Get_Owner_And_Successors_For_Device org.opendaylight.sxp.controller.boot.SxpControllerInstance
117 ... Sxp ${running_member}
120 Get Not Owner Controller
121 [Documentation] Find cluster controller that is not marked as owner for SXP service in cluster
122 ${owner_controller} = Get Owner Controller
123 ${controller} = BuiltIn.Evaluate random.choice( filter( lambda i: i!=${owner_controller}, range(1, ${NUM_ODL_SYSTEM} + 1))) random
124 [Return] ${controller}
127 [Documentation] Get any controller from cluster range
128 ${follower} = BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random
131 Map Followers To Mac Addresses
132 [Documentation] Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS
133 ${mac_addresses} = BuiltIn.Create dictionary
134 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
135 \ ${mac_address} Find Mac Address Of Ip Address ${ODL_SYSTEM_${i+1}_IP}
136 \ Collections.Set To Dictionary ${mac_addresses} ${ODL_SYSTEM_${i+1}_IP} ${mac_address}
137 BuiltIn.Log ${mac_addresses}
138 [Return] ${mac_addresses}
140 Find Mac Address Of Ip Address
142 [Documentation] Finds out MAC-ADDRESS of specified IP by pinging it from TOOLS_SYSTEM machine
143 ${mac_address} = Utils.Run Command On Remote System And Log ${TOOLS_SYSTEM_IP} ping -c 10 -W 10 ${ip} >/dev/null && sudo ip neighbor show ${ip} | awk '{print $5}' ${TOOLS_SYSTEM_USER} ${TOOLS_SYSTEM_PASSWORD}
144 [Return] ${mac_address}
146 Ip Addres Should Not Be Routed To Follower
147 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
148 [Documentation] Verify that IP-ADDRESS is not routed to follower specified by ID
149 ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP}
150 ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address}
151 BuiltIn.Should Not Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
153 Ip Addres Should Be Routed To Follower
154 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
155 [Documentation] Verify that IP-ADDRESS is routed to follower specified by ID
156 ${mac_address_assigned} = Collections.Get From Dictionary ${mac_addresses} ${ODL_SYSTEM_${follower_index}_IP}
157 ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address}
158 BuiltIn.Should Not Be Empty ${mac_address_resolved}
159 BuiltIn.Should Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
162 [Arguments] ${ip_address}=${TOOLS_SYSTEM_2_IP} ${user}=${TOOLS_SYSTEM_USER} ${passwd}=${TOOLS_SYSTEM_PASSWORD}
163 [Documentation] Shutdown Tools node to avoid conflict in resolving virtual ip that is overlaping that node.
164 ${rc} = OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address}
165 ${stdout} = BuiltIn.Run Keyword And Return If ${rc} == 0 Utils.Run Command On Remote System ${ip_address} sudo shutdown -P 0 ${user}
167 BuiltIn.Log ${stdout}
169 Create Virtual Interface
170 [Documentation] Create virtual interface on all of the cluster nodes
171 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
172 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo modprobe dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
173 \ Utils.Run Command On Remote System And Log ${ODL_SYSTEM_${i+1}_IP} sudo ip link show ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
175 Delete Virtual Interface
176 [Documentation] Create virtual interface on all of the cluster nodes
177 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
178 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo ip link delete ${VIRTUAL_INTERFACE} type dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
179 \ Utils.Run Command On Remote System ${ODL_SYSTEM_${i+1}_IP} sudo rmmod dummy ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}
180 \ Utils.Run Command On Remote System And Log ${ODL_SYSTEM_${i+1}_IP} sudo ip link show ${ODL_SYSTEM_USER} ${ODL_SYSTEM_PASSWORD}