2 Library RequestsLibrary
5 Library ClusterStateLibrary.py
6 Library ./HsfJson/hsf_json.py
10 ${jolokia_conf} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
11 ${jolokia_oper} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore
12 ${jolokia_read} /jolokia/read/org.opendaylight.controller
15 Create Controller Index List
16 [Documentation] Reads number of controllers and returns a list with all controllers indexes.
17 ${controller_index_list} Create List
18 ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM}
19 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
20 \ Append To List ${controller_index_list} ${i+1}
21 [Return] ${controller_index_list}
23 Create Controller Sessions
24 [Documentation] Creates REST session to all controller instances.
25 ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM}
26 : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM}
27 \ Log Create Session ${ODL_SYSTEM_${i+1}_IP}
28 \ RequestsLibrary.Create Session controller${i+1} http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT} auth=${AUTH}
30 Get Cluster Shard Status
31 [Arguments] ${controller_index_list} ${shard_type} ${shard}
32 [Documentation] Checks ${shard} status and returns Leader index and a list of Followers from a ${controller_index_list}.
33 ... ${shard_type} is either config or operational.
34 ${lenght}= Get Length ${controller_index_list}
35 Run Keyword If '${shard_type}' == 'config' Set Test Variable ${type} DistributedConfigDatastore
36 Run Keyword If '${shard_type}' == 'operational' Set Test Variable ${type} DistributedOperationalDatastore
37 Should Not Be Empty ${type} Wrong type, valid values are config and operational.
38 ${leader}= Set Variable 0
39 ${follower_list}= Create List
40 : FOR ${i} IN @{controller_index_list}
41 \ ${data}= Utils.Get Data From URI controller${i} ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type}
43 \ ${json}= RequestsLibrary.To Json ${data}
44 \ ${status}= Get From Dictionary &{json}[value] RaftState
45 \ Log Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard}
46 \ Run Keyword If '${status}' == 'Leader' Set Test Variable ${leader} ${i}
47 \ Run Keyword If '${status}' == 'Follower' Append To List ${follower_list} ${i}
48 Should Not Be Equal ${leader} 0 No Leader elected in shard ${shard_type} ${shard}
49 Length Should Be ${follower_list} ${lenght-1} Not enough or too many Followers in shard ${shard_type} ${shard}
50 [Return] ${leader} ${follower_list}
52 Check Item Occurrence At URI In Cluster
53 [Arguments] ${controller_index_list} ${dictionary_item_occurrence} ${uri}
54 [Documentation] Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list}
55 ... and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}.
56 : FOR ${i} IN @{controller_index_list}
57 \ ${data} Utils.Get Data From URI controller${i} ${uri}
59 \ Utils.Check Item Occurrence ${data} ${dictionary_item_occurrence}
61 Put And Check At URI In Cluster
62 [Arguments] ${controller_index_list} ${controller_index} ${uri} ${body}
63 [Documentation] Send a PUT with the supplied ${uri} and ${body} (json string) to a ${controller_index}
64 ... and check the data is replicated in all instances in ${controller_index_list}.
65 ${expected_body}= Hsf Json ${body}
67 ${resp} RequestsLibrary.Put Request controller${controller_index} ${uri} data=${body} headers=${HEADERS_YANG_JSON}
69 Log ${resp.status_code}
70 ${status_code}= Convert To String ${resp.status_code}
71 Should Match Regexp ${status_code} 20(0|1)
72 : FOR ${i} IN @{controller_index_list}
73 \ ${data} Wait Until Keyword Succeeds 5s 1s Get Data From URI controller${i}
76 \ ${received_body} Hsf Json ${data}
77 \ Should Be Equal ${expected_body} ${received_body}
79 Delete And Check At URI In Cluster
80 [Arguments] ${controller_index_list} ${controller_index} ${uri}
81 [Documentation] Send a DELETE with the supplied ${uri} to a ${controller_index}
82 ... and check the data is removed from all instances in ${controller_index_list}.
83 ${resp} RequestsLibrary.Delete Request controller${controller_index} ${uri}
84 Should Be Equal As Strings ${resp.status_code} 200
85 : FOR ${i} IN @{controller_index_list}
86 \ Wait Until Keyword Succeeds 5s 1s No Content From URI controller${i} ${uri}
88 Kill Multiple Controllers
89 [Arguments] @{controller_index_list}
90 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
91 : FOR ${i} IN @{controller_index_list}
92 \ ${output}= Utils.Run Command On Controller ${ODL_SYSTEM_${i}_IP} ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
93 \ ClusterKeywords.Controller Down Check ${ODL_SYSTEM_${i}_IP}
95 Start Multiple Controllers
96 [Arguments] ${timeout} @{controller_index_list}
97 [Documentation] Give this keyword a scalar or list of controllers to be started.
98 : FOR ${i} IN @{controller_index_list}
99 \ ${output}= Utils.Run Command On Controller ${ODL_SYSTEM_${i}_IP} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
100 \ ClusterKeywords.Wait For Controller Sync ${timeout} ${ODL_SYSTEM_${i}_IP}
103 [Arguments] ${exclude_controller}=${EMPTY}
104 [Documentation] Creates a list of all controllers minus any excluded controller.
105 Log ${exclude_controller}
106 @{searchlist} Create List ${ODL_SYSTEM_IP} ${ODL_SYSTEM_2_IP} ${ODL_SYSTEM_3_IP}
107 Remove Values From List ${searchlist} ${exclude_controller}
109 [Return] ${searchlist}
111 Get Leader And Verify
112 [Arguments] ${shard_name} ${old_leader}=${EMPTY}
113 [Documentation] Returns the IP addr or hostname of the leader of the specified shard.
114 ... Controllers are specifed in the pybot command line.
115 ${searchlist} Get Controller List ${old_leader}
116 ${leader} GetLeader ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT}
118 Should Not Be Equal As Strings ${leader} None
119 Run Keyword If '${old_leader}'!='${EMPTY}' Should Not Be Equal ${old_leader} ${leader}
123 [Arguments] ${shard_name}
124 [Documentation] No leader is elected in the car shard
125 ${leader} GetLeader ${shard_name} ${3} ${1} ${1} ${RESTCONFPORT}
126 ... ${CURRENT_CAR_LEADER}
127 Should Be Equal As Strings ${leader} None
130 [Arguments] ${shard_name} ${exclude_controller}=${EMPTY}
131 [Documentation] Returns the IP addresses or hostnames of all followers of the specified shard.
132 ${searchlist} Get Controller List ${exclude_controller}
133 ${followers} GetFollowers ${shard_name} ${3} ${3} ${1} ${RESTCONFPORT}
136 Should Not Be Empty ${followers}
137 [Return] ${followers}
139 Stop One Or More Controllers
140 [Arguments] @{controllers}
141 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
142 ${cmd} = Set Variable ${KARAF_HOME}/bin/stop
143 : FOR ${ip} IN @{controllers}
144 \ Run Command On Remote System ${ip} ${cmd}
146 Kill One Or More Controllers
147 [Arguments] @{controllers}
148 [Documentation] Give this keyword a scalar or list of controllers to be stopped.
149 ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
151 : FOR ${ip} IN @{controllers}
152 \ Run Command On Remote System ${ip} ${cmd}
154 Wait For Cluster Down
155 [Arguments] ${timeout} @{controllers}
156 [Documentation] Waits for one or more clustered controllers to be down.
157 : FOR ${ip} IN @{controllers}
158 \ ${status}= Run Keyword And Return Status Wait For Controller Down ${timeout} ${ip}
159 \ Exit For Loop If '${status}' == 'FAIL'
161 Wait For Controller Down
162 [Arguments] ${timeout} ${ip}
163 [Documentation] Waits for one controllers to be down.
164 Wait Until Keyword Succeeds ${timeout} 2s Controller Down Check ${ip}
166 Controller Down Check
168 [Documentation] Checks to see if a controller is down by verifying that the karaf process isn't present.
169 ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | wc -l
170 ${response} Run Command On COntroller ${ip} ${cmd}
171 Log Number of controller instances running: ${response}
172 Should Start With ${response} 0 Controller process found or there may be extra instances of karaf running on the host machine.
174 Start One Or More Controllers
175 [Arguments] @{controllers}
176 [Documentation] Give this keyword a scalar or list of controllers to be started.
177 ${cmd} = Set Variable ${KARAF_HOME}/bin/start
178 : FOR ${ip} IN @{controllers}
179 \ Run Command On Remote System ${ip} ${cmd}
181 Wait For Cluster Sync
182 [Arguments] ${timeout} @{controllers}
183 [Documentation] Waits for one or more clustered controllers to report Sync Status as true.
184 : FOR ${ip} IN @{controllers}
185 \ ${status}= Run Keyword And Return Status Wait For Controller Sync ${timeout} ${ip}
186 \ Exit For Loop If '${status}' == 'FAIL'
188 Wait For Controller Sync
189 [Arguments] ${timeout} ${ip}
190 [Documentation] Waits for one controllers to report Sync Status as true.
191 Wait Until Keyword Succeeds ${timeout} 2s Controller Sync Status Should Be True ${ip}
193 Controller Sync Status Should Be True
195 [Documentation] Checks if Sync Status is true.
196 ${SyncStatus}= Get Controller Sync Status ${ip}
197 Should Be Equal ${SyncStatus} ${TRUE}
199 Controller Sync Status Should Be False
201 [Documentation] Checks if Sync Status is false.
202 ${SyncStatus}= Get Controller Sync Status ${ip}
203 Should Be Equal ${SyncStatus} ${FALSE}
205 Get Controller Sync Status
206 [Arguments] ${controller_ip}
207 [Documentation] Return Sync Status.
208 Create_Session session http://${controller_ip}:${RESTCONFPORT} headers=${HEADERS} auth=${AUTH}
209 ${data}= Get Data From URI session ${jolokia_conf}
211 ${json}= To Json ${data}
212 ${value}= Get From Dictionary ${json} value
213 ${ConfSyncStatus}= Get From Dictionary ${value} SyncStatus
214 Log Configuration Sync Status: ${ConfSyncStatus}
215 ${data}= Get Data From URI session ${jolokia_oper}
217 ${json}= To Json ${data}
218 ${value}= Get From Dictionary ${json} value
219 ${OperSyncStatus}= Get From Dictionary ${value} SyncStatus
220 Log Operational Sync Status: ${OperSyncStatus}
221 Run Keyword If ${OperSyncStatus} and ${ConfSyncStatus} Set Test Variable ${SyncStatus} ${TRUE}
222 ... ELSE Set Test Variable ${SyncStatus} ${FALSE}
223 [Return] ${SyncStatus}
225 Clean One Or More Journals
226 [Arguments] @{controllers}
227 [Documentation] Give this keyword a scalar or list of controllers on which to clean journals.
228 ${del_cmd} = Set Variable rm -rf ${KARAF_HOME}/journal
229 : FOR ${ip} IN @{controllers}
230 \ Run Command On Remote System ${ip} ${del_cmd}
232 Clean One Or More Snapshots
233 [Arguments] @{controllers}
234 [Documentation] Give this keyword a scalar or list of controllers on which to clean snapshots.
235 ${del_cmd} = Set Variable rm -rf ${KARAF_HOME}/snapshots
236 : FOR ${ip} IN @{controllers}
237 \ Run Command On Remote System ${ip} ${del_cmd}
239 Show Cluster Configuation Files
240 [Arguments] @{controllers}
241 [Documentation] Prints out the cluster configuration files for one or more controllers.
242 Log controllers: @{controllers}
243 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/akka.conf
244 : FOR ${ip} IN @{controllers}
245 \ Run Command On Remote System ${ip} ${cmd}
246 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/modules.conf
247 : FOR ${ip} IN @{controllers}
248 \ Run Command On Remote System ${ip} ${cmd}
249 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/module-shards.conf
250 : FOR ${ip} IN @{controllers}
251 \ Run Command On Remote System ${ip} ${cmd}
252 ${cmd} = Set Variable cat ${KARAF_HOME}/configuration/initial/jolokia.xml
253 : FOR ${ip} IN @{controllers}
254 \ Run Command On Remote System ${ip} ${cmd}
255 ${cmd} = Set Variable cat ${KARAF_HOME}/etc/initial/org.apache.karaf.management.cfg
256 : FOR ${ip} IN @{controllers}
257 \ Run Command On Remote System ${ip} ${cmd}
258 ${cmd} = Set Variable cat ${KARAF_HOME}/etc/org.apache.karaf.features.cfg
259 : FOR ${ip} IN @{controllers}
260 \ Run Command On Remote System ${ip} ${cmd}
262 Isolate a Controller From Cluster
263 [Arguments] ${isolated controller} @{controllers}
264 [Documentation] Use IPTables to isolate one controller from the cluster.
265 ... On the isolated controller it blocks IP traffic to and from each of the other controllers.
266 : FOR ${controller} IN @{controllers}
267 \ ${other controller}= Evaluate "${isolated controller}" != "${controller}"
268 \ Run Keyword If ${other controller} Isolate One Controller From Another ${isolated controller} ${controller}
270 Rejoin a Controller To Cluster
271 [Arguments] ${isolated controller} @{controllers}
272 [Documentation] Use IPTables to rejoin one controller to the cluster.
273 ... On the isolated controller it unblocks IP traffic to and from each of the other controllers.
274 : FOR ${controller} IN @{controllers}
275 \ ${other controller}= Evaluate "${isolated controller}" != "${controller}"
276 \ Run Keyword If ${other controller} Rejoin One Controller To Another ${isolated controller} ${controller}
278 Isolate One Controller From Another
279 [Arguments] ${isolated controller} ${controller}
280 [Documentation] Inserts an IPTable rule to disconnect one controller from another controller in the cluster.
281 Modify IPTables ${isolated controller} ${controller} -I
283 Rejoin One Controller To Another
284 [Arguments] ${isolated controller} ${controller}
285 [Documentation] Deletes an IPTable rule, allowing one controller to reconnect to another controller in the cluster.
286 Modify IPTables ${isolated controller} ${controller} -D
289 [Arguments] ${isolated controller} ${controller} ${rule type}
290 [Documentation] Adds a rule, usually inserting or deleting an entry between two controllers.
291 ${base string} Set Variable sudo iptables ${rule type} OUTPUT -p all --source
292 ${cmd string} Catenate ${base string} ${isolated controller} --destination ${controller} -j DROP
293 Run Command On Remote System ${isolated controller} ${cmd string}
294 ${cmd string} Catenate ${base string} ${controller} --destination ${isolated controller} -j DROP
295 Run Command On Remote System ${isolated controller} ${cmd string}
296 ${cmd string} Set Variable sudo iptables -L -n
297 ${return string}= Run Command On Remote System ${isolated controller} ${cmd string}
299 Run Keyword If "${rule type}" == '-I' Should Match Regexp ${return string} [\s\S]*DROP *all *-- *${isolated controller} *${controller}[\s\S]*
300 Run Keyword If "${rule type}" == '-I' Should Match Regexp ${return string} [\s\S]*DROP *all *-- *${controller} *${isolated controller}[\s\S]*
302 Run Keyword If "${rule type}" == '-D' Should Match Regexp ${return string} (?![\s\S]*DROP *all *-- *${isolated controller} *${controller}[\s\S]*)
303 Run Keyword If "${rule type}" == '-D' Should Match Regexp ${return string} (?![\s\S]*DROP *all *-- *${controller} *${isolated controller}[\s\S]*)
305 Rejoin All Isolated Controllers
306 [Arguments] @{controllers}
307 [Documentation] Wipe all IPTables rules from all controllers, thus rejoining all controllers.
308 : FOR ${isolated controller} IN @{controllers}
309 \ Flush IPTables ${isolated controller}
312 [Arguments] ${isolated controller}
313 [Documentation] This keyword is generally not called from a test case but supports a complete wipe of all rules on
315 ${cmd string} Set Variable sudo iptables -v -F
316 ${return string}= Run Command On Remote System ${isolated controller} ${cmd string}
317 Log return: ${return string}
318 Should Contain ${return string} Flushing chain `INPUT'
319 Should Contain ${return string} Flushing chain `FORWARD'
320 Should Contain ${return string} Flushing chain `OUTPUT'