2 Documentation Library containing Keywords used for SXP cluster testing
5 Library RequestsLibrary
6 Resource ./ClusterManagement.robot
7 Resource ./CompareStream.robot
8 Resource ./SetupUtils.robot
9 Resource ./SxpLib.robot
13 @{SHARD_OPER_LIST} inventory topology default entity-ownership
14 @{SHARD_CONF_LIST} inventory topology default
15 @{SXP_PACKAGE} org.opendaylight.sxp
16 ${DEVICE_SESSION} device_1
17 ${CONTROLLER_SESSION} ClusterManagement__session_1
18 ${VIRTUAL_IP} ${TOOLS_SYSTEM_2_IP}
19 ${VIRTUAL_IP_MASK} 255.255.255.0
20 ${VIRTUAL_INTERFACE} dummy0
21 ${MAC_ADDRESS_TABLE} &{EMPTY}
22 ${DEVICE_NODE_ID} ${TOOLS_SYSTEM_IP}
23 ${CLUSTER_NODE_ID} ${TOOLS_SYSTEM_2_IP}
28 Setup SXP Cluster Session
29 [Documentation] Create sessions asociated with SXP cluster setup
30 ClusterManagement.ClusterManagement_Setup
31 SetupUtils.Setup_Utils_For_Setup_And_Teardown
32 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All DEBUG ${SXP_PACKAGE}
35 [Documentation] Create session on the SXP device
36 RequestsLibrary.Create Session
38 ... url=http://${DEVICE_NODE_ID}:${RESTCONFPORT}
40 ... timeout=${DEFAULT_TIMEOUT_HTTP}
43 Setup SXP Cluster Session With Device
44 [Documentation] Create sessions asociated with SXP cluster setup and one SXP device
45 Setup SXP Cluster Session
48 Clean SXP Cluster Session
49 [Documentation] Clean sessions asociated with SXP cluster setup
50 ClusterManagement.Flush_Iptables_From_List_Or_All
51 BuiltIn.Wait Until Keyword Succeeds
54 ... ClusterManagement.Verify_Members_Are_Ready
55 ... member_index_list=${EMPTY}
56 ... verify_cluster_sync=True
57 ... verify_restconf=True
58 ... verify_system_status=False
59 ... service_list=@{EMPTY}
60 RequestsLibrary.Delete All Sessions
61 SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All INFO ${SXP_PACKAGE}
64 [Documentation] Check Status for all shards in SXP application.
65 ClusterManagement.Check_Cluster_Is_In_Sync
66 CompareStream.Run_Keyword_If_At_Least_Phosphorus
67 ... Collections.Remove Values From List
68 ... ${SHARD_OPER_LIST}
70 Log ${SHARD_OPER_LIST}
71 ClusterManagement.Verify_Leader_Exists_For_Each_Shard
72 ... shard_name_list=${SHARD_OPER_LIST}
73 ... shard_type=operational
74 ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_CONF_LIST} shard_type=config
77 [Documentation] Setup and connect SXP cluster topology
78 [Arguments] ${peer_mode}=listener
79 SxpLib.Add Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
80 BuiltIn.Wait Until Keyword Succeeds
83 ... SxpLib.Check Node Started
85 ... session=${DEVICE_SESSION}
86 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
87 SxpLib.Add Connection version4 ${peer_mode} ${ODL_SYSTEM_${i+1}_IP} 64999 node=${DEVICE_NODE_ID}
88 ... session=${DEVICE_SESSION}
90 ${cluster_mode} = Sxp.Get Opposing Mode ${peer_mode}
91 SxpLib.Add Node ${INADDR_ANY} session=${CONTROLLER_SESSION}
92 BuiltIn.Wait Until Keyword Succeeds 240x 1s Check Cluster Node Started ${INADDR_ANY} ip=${EMPTY}
99 ... session=${CONTROLLER_SESSION}
100 BuiltIn.Wait Until Keyword Succeeds
103 ... Check Device is Connected
104 ... ${DEVICE_NODE_ID}
105 ... session=${DEVICE_SESSION}
108 [Documentation] Disconnect SXP cluster topology
109 SxpLib.Delete Node ${DEVICE_NODE_ID} session=${DEVICE_SESSION}
110 BuiltIn.Wait Until Keyword Succeeds
113 ... SxpLib.Check Node Stopped
114 ... ${DEVICE_NODE_ID}
115 ... session=${DEVICE_SESSION}
116 BuiltIn.Wait Until Keyword Succeeds
119 ... SxpLib.Delete Node
121 ... session=${CONTROLLER_SESSION}
122 BuiltIn.Wait Until Keyword Succeeds
125 ... SxpClusterLib.Check Cluster Node Stopped
129 Check Cluster Node Started
130 [Documentation] Verify that SxpNode has data written to Operational datastore and Node is running on one of cluster nodes
131 [Arguments] ${node} ${port}=64999 ${ip}=${node}
132 ${resp} = RequestsLibrary.Get Request
133 ... ${CONTROLLER_SESSION}
134 ... /restconf/operational/network-topology:network-topology/topology/sxp/node/${node}/
135 BuiltIn.Should Be Equal As Strings ${resp.status_code} 200
136 ${started} = BuiltIn.Set Variable ${False}
137 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
138 ${rc} = Utils.Run Command On Remote System
139 ... ${ODL_SYSTEM_${i+1}_IP}
140 ... netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1
141 ... ${ODL_SYSTEM_USER}
142 ... ${ODL_SYSTEM_PASSWORD}
143 ... prompt=${ODL_SYSTEM_PROMPT}
144 ${started} = BuiltIn.Set Variable If '${rc}' == '0' ${True} ${started}
146 BuiltIn.Should Be True ${started}
148 Check Cluster Node Stopped
149 [Documentation] Verify that SxpNode has data removed from Operational datastore and Node is stopped
150 [Arguments] ${node} ${port}=64999 ${ip}=${node}
151 ${resp} = RequestsLibrary.Get Request
152 ... ${CONTROLLER_SESSION}
153 ... /restconf/operational/network-topology:network-topology/topology/sxp/node/${node}/
154 BuiltIn.Should Be Equal As Strings ${resp.status_code} 404
155 ${stopped} = BuiltIn.Set Variable ${False}
156 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
157 ${rc} = Utils.Run Command On Remote System
158 ... ${ODL_SYSTEM_${i+1}_IP}
159 ... netstat -tln | grep -q ${ip}:${port} && echo 0 || echo 1
160 ... ${ODL_SYSTEM_USER}
161 ... ${ODL_SYSTEM_PASSWORD}
162 ... prompt=${ODL_SYSTEM_PROMPT}
163 ${stopped} = BuiltIn.Set Variable If '${rc}' == '1' ${True} ${stopped}
165 BuiltIn.Should Be True ${stopped}
167 Check Device is Connected
168 [Documentation] Checks if SXP device is connected to the cluster. It means it has connection in state "on" with one of the cluster members.
169 [Arguments] ${node} ${version}=version4 ${port}=64999 ${session}=session
170 ${resp} = SxpLib.Get Connections node=${node} session=${session}
171 ${is_connected} = BuiltIn.Set Variable ${False}
172 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
173 ${is_connected} = Sxp.Find Connection ${resp} ${version} any ${ODL_SYSTEM_${i+1}_IP}
175 IF ${is_connected} BREAK
177 BuiltIn.Should Be True ${is_connected}
179 Check Cluster is Connected
180 [Documentation] Get SXP connections of cluster ${node} and verify that they contain a connection to the device ${DEVICE_NODE_ID} in state "on"
181 [Arguments] ${node} ${version}=version4 ${port}=64999 ${mode}=speaker ${session}=session
182 ${resp} = SxpLib.Get Connections node=${node} session=${session}
183 SxpLib.Should Contain Connection ${resp} ${DEVICE_NODE_ID} ${port} ${mode} ${version} on
186 [Documentation] Find cluster controller that is marked as cluster owner by requesting ownership data from ${running_member} node of the cluster
187 [Arguments] ${running_member}=1
188 ${owner} ${candidates} = BuiltIn.Wait Until Keyword Succeeds
191 ... ClusterManagement.Get_Owner_And_Successors_For_Device
192 ... org.opendaylight.sxp.controller.boot.SxpControllerInstance
194 ... ${running_member}
197 Get Not Owner Controller
198 [Documentation] Find cluster controller that is not marked as owner for SXP service in cluster
199 ${owner_controller} = Get Owner Controller
200 ${controller} = BuiltIn.Evaluate
201 ... random.choice( filter( lambda i: i!=${owner_controller}, range(1, ${NUM_ODL_SYSTEM} + 1)))
206 [Documentation] Get any controller from cluster range
207 ${follower} = BuiltIn.Evaluate random.choice( range(1, ${NUM_ODL_SYSTEM} + 1)) random
210 Map Followers To Mac Addresses
211 [Documentation] Creates Map containing ODL_SYSTEM_IP to corresponding MAC-ADDRESS
212 ${mac_addresses} = BuiltIn.Create dictionary
213 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
214 ${mac_address} = Find Mac Address Of Ip Address ${ODL_SYSTEM_${i+1}_IP}
215 Collections.Set To Dictionary ${mac_addresses} ${ODL_SYSTEM_${i+1}_IP} ${mac_address}
217 BuiltIn.Log ${mac_addresses}
218 RETURN ${mac_addresses}
220 Find Mac Address Of Ip Address
221 [Documentation] Finds out MAC-ADDRESS of specified IP by pinging it from TOOLS_SYSTEM machine
223 ${mac_address} = Utils.Run Command On Remote System And Log
224 ... ${TOOLS_SYSTEM_IP}
225 ... ping -c 10 -W 10 ${ip} >/dev/null && sudo ip neighbor show ${ip} | awk '{print $5}'
226 ... ${TOOLS_SYSTEM_USER}
227 ... ${TOOLS_SYSTEM_PASSWORD}
228 RETURN ${mac_address}
230 Ip Addres Should Not Be Routed To Follower
231 [Documentation] Verify that IP-ADDRESS is not routed to follower specified by ID
232 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
233 ${mac_address_assigned} = Collections.Get From Dictionary
235 ... ${ODL_SYSTEM_${follower_index}_IP}
236 ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address}
237 BuiltIn.Should Not Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
239 Ip Addres Should Be Routed To Follower
240 [Documentation] Verify that IP-ADDRESS is routed to follower specified by ID
241 [Arguments] ${mac_addresses} ${ip_address} ${follower_index}
242 ${mac_address_assigned} = Collections.Get From Dictionary
244 ... ${ODL_SYSTEM_${follower_index}_IP}
245 ${mac_address_resolved} = Find Mac Address Of Ip Address ${ip_address}
246 BuiltIn.Should Not Be Empty ${mac_address_resolved}
247 BuiltIn.Should Be Equal As Strings ${mac_address_assigned} ${mac_address_resolved}
250 [Documentation] Shutdown Tools node to avoid conflict in resolving virtual ip that is overlaping that node.
251 [Arguments] ${ip_address}=${TOOLS_SYSTEM_2_IP} ${user}=${TOOLS_SYSTEM_USER} ${passwd}=${TOOLS_SYSTEM_PASSWORD}
252 ${rc} = OperatingSystem.Run And Return Rc ping -q -c 3 ${ip_address}
253 ${stdout} = BuiltIn.Run Keyword And Return If
255 ... Utils.Run Command On Remote System
257 ... sudo shutdown -P 0
260 BuiltIn.Log ${stdout}
262 Create Virtual Interface
263 [Documentation] Create virtual interface on all of the cluster nodes
264 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
265 Utils.Run Command On Remote System
266 ... ${ODL_SYSTEM_${i+1}_IP}
267 ... sudo modprobe dummy
268 ... ${ODL_SYSTEM_USER}
269 ... ${ODL_SYSTEM_PASSWORD}
270 Utils.Run Command On Remote System And Log
271 ... ${ODL_SYSTEM_${i+1}_IP}
272 ... sudo ip link show
273 ... ${ODL_SYSTEM_USER}
274 ... ${ODL_SYSTEM_PASSWORD}
277 Delete Virtual Interface
278 [Documentation] Create virtual interface on all of the cluster nodes
279 FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
280 Utils.Run Command On Remote System
281 ... ${ODL_SYSTEM_${i+1}_IP}
282 ... sudo ip link delete ${VIRTUAL_INTERFACE} type dummy
283 ... ${ODL_SYSTEM_USER}
284 ... ${ODL_SYSTEM_PASSWORD}
285 Utils.Run Command On Remote System
286 ... ${ODL_SYSTEM_${i+1}_IP}
288 ... ${ODL_SYSTEM_USER}
289 ... ${ODL_SYSTEM_PASSWORD}
290 Utils.Run Command On Remote System And Log
291 ... ${ODL_SYSTEM_${i+1}_IP}
292 ... sudo ip link show
293 ... ${ODL_SYSTEM_USER}
294 ... ${ODL_SYSTEM_PASSWORD}