2 Library RequestsLibrary
5 Library ClusterStateLibrary.py
9 ${jolokia_conf} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
10 ${jolokia_oper} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore
11 ${jolokia_read} /jolokia/read/org.opendaylight.controller
14 Create Controller Index List
15 [Documentation] Reads number of controllers and returns a list with all controllers indexes.
16 ${controller_index_list} Create List
17 ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM}
18 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
19 \ Append To List ${controller_index_list} ${i+1}
20 [Return] ${controller_index_list}
22 Create Controller Sessions
23 [Documentation] Creates REST session to all controller instances.
24 ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM}
25 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
26 \ Log Create Session ${ODL_SYSTEM_${i+1}_IP}
27 \ Create Session controller${i+1} http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT} auth=${AUTH}
29 Get Cluster Shard Status
30 [Arguments] ${controller_index_list} ${shard_type} ${shard}
31 [Documentation] Checks ${shard} status and returns Leader index and a list of Followers from a ${controller_index_list}.
32 ... ${shard_type} is either config or operational.
33 ${lenght}= Get Length ${controller_index_list}
34 Run Keyword If '${shard_type}' == 'config' Set Test Variable ${type} DistributedConfigDatastore
35 Run Keyword If '${shard_type}' == 'operational' Set Test Variable ${type} DistributedOperationalDatastore
36 Should Not Be Empty ${type} Wrong type, valid values are config and operational.
37 ${leader}= Set Variable 0
38 ${follower_list}= Create List
39 : FOR ${i} IN @{controller_index_list}
40 \ ${data}= Get Data From URI controller${i} ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type}
42 \ ${json}= To Json ${data}
43 \ ${status}= Get From Dictionary &{json}[value] RaftState
44 \ Log Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard}
45 \ Run Keyword If '${status}' == 'Leader' Set Test Variable ${leader} ${i}
46 \ Run Keyword If '${status}' == 'Follower' Append To List ${follower_list} ${i}
47 Should Not Be Equal ${leader} 0 No Leader elected in shard ${shard_type} ${shard}
48 Length Should Be ${follower_list} ${lenght-1} Not enough or too many Followers in shard ${shard_type} ${shard}
49 [Return] ${leader} ${follower_list}
51 Get Cluster Entity Owner Status
52 [Arguments] ${controller_index_list} ${device_type} ${device}
53 [Documentation] Checks Entity Owner status for a ${device} and returns owner index and list of candidates from a ${controller_index_list}.
54 ... ${device_type} is openflow, ovsdb, etc...
55 ${length}= Get Length ${controller_index_list}
56 ${candidates_list}= Create List
57 ${data}= Get Data From URI controller@{controller_index_list}[0] /restconf/operational/entity-owners:entity-owners
59 ${data}= Replace String ${data} /general-entity:entity[general-entity:name=' ${EMPTY}
60 ${clear_data}= Replace String ${data} '] ${EMPTY}
62 ${json}= To Json ${clear_data}
63 ${entity_type_list}= Get From Dictionary &{json}[entity-owners] entity-type
64 ${entity_type_index}= Get Index From List Of Dictionaries ${entity_type_list} type ${device_type}
65 Should Not Be Equal ${entity_type_index} -1 No Entity Owner found for ${device_type}
66 ${entity_list}= Get From Dictionary @{entity_type_list}[${entity_type_index}] entity
67 ${entity_index}= Get Index From List Of Dictionaries ${entity_list} id ${device}
68 Should Not Be Equal ${entity_index} -1 Device ${device} not found in Entity Owner ${device_type}
69 ${entity_owner}= Get From Dictionary @{entity_list}[${entity_index}] owner
70 Should Not Be Empty ${entity_owner} No owner found for ${device}
71 ${owner}= Replace String ${entity_owner} member- ${EMPTY}
72 ${owner}= Convert To Integer ${owner}
73 List Should Contain Value ${controller_index_list} ${owner} Owner ${owner} not exisiting in ${controller_index_list}
74 ${entity_candidates_list}= Get From Dictionary @{entity_list}[${entity_index}] candidate
75 ${list_length}= Get Length ${entity_candidates_list}
76 : FOR ${entity_candidate} IN @{entity_candidates_list}
77 \ ${candidate}= Replace String &{entity_candidate}[name] member- ${EMPTY}
78 \ ${candidate}= Convert To Integer ${candidate}
79 \ List Should Contain Value ${controller_index_list} ${candidate} Candidate ${candidate} not exisiting in ${controller_index_list}
80 \ Run Keyword If '${candidate}' != '${owner}' Append To List ${candidates_list} ${candidate}
81 [Return] ${owner} ${candidates_list}
83 Check Item Occurrence At URI In Cluster
84 [Arguments] ${controller_index_list} ${dictionary_item_occurrence} ${uri} ${headers}=${HEADERS}
85 [Documentation] Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list}
86 ... and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}.
87 : FOR ${i} IN @{controller_index_list}
88 \ ${data} Get Data From URI controller${i} ${uri} ${headers}
90 \ Check Item Occurrence ${data} ${dictionary_item_occurrence}
92 Put And Check At URI In Cluster
93 [Arguments] ${controller_index_list} ${controller_index} ${uri} ${body} ${headers}=${HEADERS}
94 [Documentation] Send a PUT with the supplied ${uri} and ${body} to a ${controller_index}
95 ... and check the data is replicated in all instances in ${controller_index_list}.
96 ${expected_body}= To Json ${body}
97 ${resp} RequestsLibrary.Put Request controller${controller_index} ${uri} ${body} ${headers}
98 Should Be Equal As Strings ${resp.status_code} 200
99 : FOR ${i} IN @{controller_index_list}
100 \ ${data} Wait Until Keyword Succeeds 5s 1s Get Data From URI controller${i}
101 \ ... ${uri} ${headers}
103 \ ${received_body} To Json ${data}
104 \ Should Be Equal ${received_body} ${expected_body}
106 Delete And Check At URI In Cluster
107 [Arguments] ${controller_index_list} ${controller_index} ${uri} ${headers}=${HEADERS}
108 [Documentation] Send a DELETE with the supplied ${uri} to a ${controller_index}
109 ... and check the data is removed from all instances in ${controller_index_list}.
110 ${resp} RequestsLibrary.Delete Request controller${controller_index} ${uri} ${headers}
111 Should Be Equal As Strings ${resp.status_code} 200
112 : FOR ${i} IN @{controller_index_list}
113 \ Wait Until Keyword Succeeds 5s 1s No Content From URI controller${i} ${uri}
116 Kill Multiple Controllers
117 [Arguments] @{controller_index_list}
118 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
119 : FOR ${i} IN @{controller_index_list}
120 \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
121 \ Controller Down Check ${ODL_SYSTEM_${i}_IP}
123 Start Multiple Controllers
124 [Arguments] ${timeout} @{controller_index_list}
125 [Documentation] Give this keyword a scalar or list of controllers to be started.
126 : FOR ${i} IN @{controller_index_list}
127 \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
128 \ Wait For Controller Sync ${timeout} ${ODL_SYSTEM_${i}_IP}
131 [Arguments] ${exclude_controller}=${EMPTY}
132 [Documentation] Creates a list of all controllers minus any excluded controller.
133 Log ${exclude_controller}
134 @{searchlist} Create List ${CONTROLLER} ${CONTROLLER1} ${CONTROLLER2}
135 Remove Values From List ${searchlist} ${exclude_controller}
137 [Return] ${searchlist}
139 Get Leader And Verify
140 [Arguments] ${shard_name} ${old_leader}=${EMPTY}
141 [Documentation] Returns the IP addr or hostname of the leader of the specified shard.
142 ... Controllers are specifed in the pybot command line.
143 ${searchlist} Get Controller List ${old_leader}
144 ${leader} GetLeader ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT}
146 Should Not Be Equal As Strings ${leader} None
147 Run Keyword If '${old_leader}'!='${EMPTY}' Should Not Be Equal ${old_leader} ${leader}
151 [Arguments] ${shard_name}
152 [Documentation] No leader is elected in the car shard
153 ${leader} GetLeader ${shard_name} ${3} ${1} ${1} ${RESTCONFPORT}
154 ... ${CURRENT_CAR_LEADER}
155 Should Be Equal As Strings ${leader} None
158 [Arguments] ${shard_name} ${exclude_controller}=${EMPTY}
159 [Documentation] Returns the IP addresses or hostnames of all followers of the specified shard.
160 ${searchlist} Get Controller List ${exclude_controller}
161 ${followers} GetFollowers ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT}
164 Should Not Be Empty ${followers}
165 [Return] ${followers}
167 Stop One Or More Controllers
168 [Arguments] @{controllers}
169 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
170 ${cmd} = Set Variable ${KARAF_HOME}/bin/stop
171 : FOR ${ip} IN @{controllers}
172 \ Run Command On Remote System ${ip} ${cmd}
174 Kill One Or More Controllers
175 [Arguments] @{controllers}
176 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
177 ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
179 : FOR ${ip} IN @{controllers}
180 \ Run Command On Remote System ${ip} ${cmd}
182 Wait For Cluster Down
183 [Arguments] ${timeout} @{controllers}
184 [Documentation] Waits for one or more clustered controllers to be down.
185 : FOR ${ip} IN @{controllers}
186 \ ${status}= Run Keyword And Return Status Wait For Controller Down ${timeout} ${ip}
187 \ Exit For Loop If '${status}' == 'FAIL'
189 Wait For Controller Down
190 [Arguments] ${timeout} ${ip}
191 [Documentation] Waits for one controllers to be down.
192 Wait Until Keyword Succeeds ${timeout} 2s Controller Down Check ${ip}
194 Controller Down Check
196 [Documentation] Checks to see if a controller is down by verifying that the karaf process isn't present.
197 ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | wc -l
198 ${response} Run Command On COntroller ${ip} ${cmd}
199 Log Number of controller instances running: ${response}
200 Should Start With ${response} 0 Controller process found or there may be extra instances of karaf running on the host machine.
202 Start One Or More Controllers
203 [Arguments] @{controllers}
204 [Documentation] Give this keyword a scalar or list of controllers to be started.
205 ${cmd} = Set Variable ${KARAF_HOME}/bin/start
206 : FOR ${ip} IN @{controllers}
207 \ Run Command On Remote System ${ip} ${cmd}
209 Wait For Cluster Sync
210 [Arguments] ${timeout} @{controllers}
211 [Documentation] Waits for one or more clustered controllers to report Sync Status as true.
212 : FOR ${ip} IN @{controllers}
213 \ ${status}= Run Keyword And Return Status Wait For Controller Sync ${timeout} ${ip}
214 \ Exit For Loop If '${status}' == 'FAIL'
216 Wait For Controller Sync
217 [Arguments] ${timeout} ${ip}
218 [Documentation] Waits for one controllers to report Sync Status as true.
219 Wait Until Keyword Succeeds ${timeout} 2s Controller Sync Status Should Be True ${ip}
221 Controller Sync Status Should Be True
223 [Documentation] Checks if Sync Status is true.
224 ${SyncStatus}= Get Controller Sync Status ${ip}
225 Should Be Equal ${SyncStatus} ${TRUE}
227 Controller Sync Status Should Be False
229 [Documentation] Checks if Sync Status is false.
230 ${SyncStatus}= Get Controller Sync Status ${ip}
231 Should Be Equal ${SyncStatus} ${FALSE}
233 Get Controller Sync Status
234 [Arguments] ${controller_ip}
235 [Documentation] Return Sync Status.
236 Create_Session session http://${controller_ip}:${RESTCONFPORT} headers=${HEADERS} auth=${AUTH}
237 ${data}= Get Data From URI session ${jolokia_conf}
239 ${json}= To Json ${data}
240 ${value}= Get From Dictionary ${json} value
241 ${ConfSyncStatus}= Get From Dictionary ${value} SyncStatus
242 Log Configuration Sync Status: ${ConfSyncStatus}
243 ${data}= Get Data From URI session ${jolokia_oper}
245 ${json}= To Json ${data}
246 ${value}= Get From Dictionary ${json} value
247 ${OperSyncStatus}= Get From Dictionary ${value} SyncStatus
248 Log Operational Sync Status: ${OperSyncStatus}
249 Run Keyword If ${OperSyncStatus} and ${ConfSyncStatus} Set Test Variable ${SyncStatus} ${TRUE}
250 ... ELSE Set Test Variable ${SyncStatus} ${FALSE}
251 [Return] ${SyncStatus}
253 Clean One Or More Journals
254 [Arguments] @{controllers}
255 [Documentation] Give this keyword a scalar or list of controllers on which to clean journals.
256 ${del_cmd} = Set Variable rm -rf ${KARAF_HOME}/journal
257 : FOR ${ip} IN @{controllers}
258 \ Run Command On Remote System ${ip} ${del_cmd}
260 Clean One Or More Snapshots
261 [Arguments] @{controllers}
262 [Documentation] Give this keyword a scalar or list of controllers on which to clean snapshots.
263 ${del_cmd} = Set Variable rm -rf ${KARAF_HOME}/snapshots
264 : FOR ${ip} IN @{controllers}
265 \ Run Command On Remote System ${ip} ${del_cmd}
267 Show Cluster Configuation Files
268 [Arguments] @{controllers}
269 [Documentation] Prints out the cluster configuration files for one or more controllers.
270 Log controllers: @{controllers}
271 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/akka.conf
272 : FOR ${ip} IN @{controllers}
273 \ Run Command On Remote System ${ip} ${cmd}
274 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/modules.conf
275 : FOR ${ip} IN @{controllers}
276 \ Run Command On Remote System ${ip} ${cmd}
277 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/module-shards.conf
278 : FOR ${ip} IN @{controllers}
279 \ Run Command On Remote System ${ip} ${cmd}
280 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/jolokia.xml
281 : FOR ${ip} IN @{controllers}
282 \ Run Command On Remote System ${ip} ${cmd}
283 ${cmd} = Set Variable cat ${KARAF_HOME}/etc/initial/org.apache.karaf.management.cfg
284 : FOR ${ip} IN @{controllers}
285 \ Run Command On Remote System ${ip} ${cmd}
286 ${cmd} = Set Variable cat ${KARAF_HOME}/etc/org.apache.karaf.features.cfg
287 : FOR ${ip} IN @{controllers}
288 \ Run Command On Remote System ${ip} ${cmd}
290 Isolate a Controller From Cluster
291 [Arguments] ${isolated controller} @{controllers}
292 [Documentation] Use IPTables to isolate one controller from the cluster.
293 ... On the isolated controller it blocks IP traffic to and from each of the other controllers.
294 : FOR ${controller} IN @{controllers}
295 \ ${other controller}= Evaluate "${isolated controller}" != "${controller}"
296 \ Run Keyword If ${other controller} Isolate One Controller From Another ${isolated controller} ${controller}
298 Rejoin a Controller To Cluster
299 [Arguments] ${isolated controller} @{controllers}
300 [Documentation] Use IPTables to rejoin one controller to the cluster.
301 ... On the isolated controller it unblocks IP traffic to and from each of the other controllers.
302 : FOR ${controller} IN @{controllers}
303 \ ${other controller}= Evaluate "${isolated controller}" != "${controller}"
304 \ Run Keyword If ${other controller} Rejoin One Controller To Another ${isolated controller} ${controller}
306 Isolate One Controller From Another
307 [Arguments] ${isolated controller} ${controller}
308 [Documentation] Inserts an IPTable rule to disconnect one controller from another controller in the cluster.
309 Modify IPTables ${isolated controller} ${controller} -I
311 Rejoin One Controller To Another
312 [Arguments] ${isolated controller} ${controller}
313 [Documentation] Deletes an IPTable rule, allowing one controller to reconnect to another controller in the cluster.
314 Modify IPTables ${isolated controller} ${controller} -D
317 [Arguments] ${isolated controller} ${controller} ${rule type}
318 [Documentation] Adds a rule, usually inserting or deleting an entry between two controllers.
319 ${base string} Set Variable sudo iptables ${rule type} OUTPUT -p all --source
320 ${cmd string} Catenate ${base string} ${isolated controller} --destination ${controller} -j DROP
321 Run Command On Remote System ${isolated controller} ${cmd string}
322 ${cmd string} Catenate ${base string} ${controller} --destination ${isolated controller} -j DROP
323 Run Command On Remote System ${isolated controller} ${cmd string}
324 ${cmd string} Set Variable sudo iptables -L -n
325 ${return string}= Run Command On Remote System ${isolated controller} ${cmd string}
327 Run Keyword If "${rule type}" == '-I' Should Match Regexp ${return string} [\s\S]*DROP *all *-- *${isolated controller} *${controller}[\s\S]*
328 Run Keyword If "${rule type}" == '-I' Should Match Regexp ${return string} [\s\S]*DROP *all *-- *${controller} *${isolated controller}[\s\S]*
330 Run Keyword If "${rule type}" == '-D' Should Match Regexp ${return string} (?![\s\S]*DROP *all *-- *${isolated controller} *${controller}[\s\S]*)
331 Run Keyword If "${rule type}" == '-D' Should Match Regexp ${return string} (?![\s\S]*DROP *all *-- *${controller} *${isolated controller}[\s\S]*)
333 Rejoin All Isolated Controllers
334 [Arguments] @{controllers}
335 [Documentation] Wipe all IPTables rules from all controllers, thus rejoining all controllers.
336 : FOR ${isolated controller} IN @{controllers}
337 \ Flush IPTables ${isolated controller}
340 [Arguments] ${isolated controller}
341 [Documentation] This keyword is generally not called from a test case but supports a complete wipe of all rules on
343 ${cmd string} Set Variable sudo iptables -v -F
344 ${return string}= Run Command On Remote System ${isolated controller} ${cmd string}
345 Log return: ${return string}
346 Should Contain ${return string} Flushing chain `INPUT'
347 Should Contain ${return string} Flushing chain `FORWARD'
348 Should Contain ${return string} Flushing chain `OUTPUT'