2 Documentation DOMDataBroker testing: Common keywords
4 ... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
6 ... This program and the accompanying materials are made available under the
7 ... terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 ... and is available at http://www.eclipse.org/legal/epl-v10.html
10 ... This resource file implements various test cases templates.
11 ... FIXME: add a link to a document (when published) where the scenarios are defined
13 ... TODO: When checking first response in isolation scenarior, make sure it comes from the expected member.
15 Library ${CURDIR}/../MdsalLowlevelPy.py
16 Resource ${CURDIR}/../ClusterAdmin.robot
17 Resource ${CURDIR}/../ClusterManagement.robot
18 Resource ${CURDIR}/../KarafKeywords.robot
19 Resource ${CURDIR}/../MdsalLowlevel.robot
20 Resource ${CURDIR}/../TemplatedRequests.robot
21 Resource ${CURDIR}/../ShardStability.robot
22 Resource ${CURDIR}/../WaitForFailure.robot
28 ${TRANSACTION_RATE_1K} ${1000}
29 ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT} ${2*${REQUEST_TIMEOUT}}
30 ${TRANSACTION_PRODUCTION_TIME} ${40}
31 ${SLEEP_AFTER_TRANSACTIONS_INIT} 5s
33 # different-prefix- has been used before, but currently is neither needed nor supported
38 ${ISOLATED_TRANS_TRUE} ${True}
39 ${ISOLATED_TRANS_FALSE} ${False}
40 ${JAVA_INTERNAL_RECONNECT_TIMEOUT} ${30}
41 ${REQUEST_TIMEOUT} ${120}
42 ${HEAL_WITHIN_REQUEST_TIMEOUT} ${${JAVA_INTERNAL_RECONNECT_TIMEOUT}+10}
43 ${HEAL_AFTER_REQUEST_TIMEOUT} ${${REQUEST_TIMEOUT}+10}
44 @{TRANSACTION_FAILED} ${500}
45 ${PREF_BASED_SHARD} id-ints
46 ${TEST_LOG_LEVEL} info
47 @{TEST_LOG_COMPONENTS}
48 ... org.opendaylight.controller.cluster.sharding
49 ... org.opendaylight.controller.cluster.datastore
50 ${HEAL_WITHIN_TRANS_TIMEOUT} ${0}
54 Explicit_Leader_Movement_Test_Templ
55 [Documentation] Implements explicit leader movement test scenario.
56 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
57 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test
62 KarafKeywords.Log_Message_To_Controller_Karaf
63 ... Starting leader movement from node${idx_from} to node${idx_to}, transaction producer at node${idx_trans}.
64 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
65 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
66 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
67 ... ${ip_trans_as_list}
68 ... ${idx_trans_as_list}
70 ... ${TRANSACTION_PRODUCTION_TIME}
71 ... ${TRANSACTION_RATE_1K}
72 ... chained_flag=${CHAINED_TX}
73 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
74 ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type}
75 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds
78 ... ClusterManagement.Verify_Shard_Leader_Elected
83 ... verify_restconf=False
84 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
85 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
86 Check_Status_Of_First_Response ${resp_list}
88 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
89 [Documentation] Implements explicit leader movement test scenario.
90 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
91 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test
96 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
97 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
98 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
99 ... ${ip_trans_as_list}
100 ... ${idx_trans_as_list}
102 ... ${TRANSACTION_PRODUCTION_TIME}
103 ... ${TRANSACTION_RATE_1K}
104 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
105 MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name}
106 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds
109 ... ClusterManagement.Verify_Shard_Leader_Elected
114 ... verify_restconf=False
115 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
116 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
117 Check_Status_Of_First_Response ${resp_list}
119 Get_Node_Indexes_For_The_ELM_Test
120 [Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
121 ... producer should be deployed.
122 [Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type}
123 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard
124 ... shard_name=${shard_name}
125 ... shard_type=${shard_type}
126 ... verify_restconf=False
127 ${idx_from} = BuiltIn.Set_Variable ${leader}
128 ${idx_to} = BuiltIn.Set_Variable ${follower_list}[0]
129 ${idx_trans} = BuiltIn.Set_Variable_If
130 ... "${leader_from}" == "remote" and "${leader_to}" == "remote"
131 ... ${follower_list}[1]
132 ... "${leader_from}" == "local"
134 ... "${leader_to}" == "local"
135 ... ${follower_list}[0]
136 RETURN ${idx_from} ${idx_to} ${idx_trans}
138 Clean_Leader_Shutdown_Test_Templ
139 [Documentation] Implements clean leader shutdown test scenario.
140 [Arguments] ${leader_location} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
141 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
142 ... ${leader_location}
145 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
146 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
147 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
148 ... ${producer_ip_as_list}
149 ... ${producer_idx_as_list}
151 ... ${TRANSACTION_PRODUCTION_TIME}
152 ... ${TRANSACTION_RATE_1K}
153 ... chained_flag=${CHAINED_TX}
154 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
155 BuiltIn.Comment Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
156 ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
157 BuiltIn.Wait_Until_Keyword_Succeeds
160 ... Verify_Shard_Replica_Not_Present
164 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
165 Check_Status_Of_First_Response ${resp_list}
166 [Teardown] BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 120s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
167 ... shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
169 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
170 [Documentation] Implements clean leader shutdown test scenario.
171 [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
172 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
173 ... ${leader_location}
176 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
177 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
178 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
179 ... ${producer_ip_as_list}
180 ... ${producer_idx_as_list}
182 ... ${TRANSACTION_PRODUCTION_TIME}
183 ... ${TRANSACTION_RATE_1K}
184 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
185 BuiltIn.Comment Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
186 ClusterAdmin.Remove_Prefix_Shard_Replica
189 ... member-${actual_leader}
191 BuiltIn.Wait_Until_Keyword_Succeeds
194 ... Verify_Shard_Replica_Not_Present
198 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
199 Check_Status_Of_First_Response ${resp_list}
200 [Teardown] BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 120s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
201 ... shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
203 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
204 [Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
205 [Arguments] ${leader_location} ${shard_name} ${shard_type}
206 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard
207 ... shard_name=${shard_name}
208 ... shard_type=${shard_type}
209 ... verify_restconf=False
210 ${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1
211 BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth}
212 ${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} ${follower_list}[0]
213 RETURN ${producer_idx} ${leader} ${follower_list}
215 Leader_Isolation_Test_Templ
216 [Documentation] Implements leader isolation test scenario.
217 [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
218 ${li_isolated} = BuiltIn.Set_Variable ${False}
219 ${producing_transactions_time} = BuiltIn.Set_Variable_If
220 ... ${heal_timeout}<${REQUEST_TIMEOUT}
221 ... ${${REQUEST_TIMEOUT}+60}
222 ... ${2*${REQUEST_TIMEOUT}}
223 ${all_indices} = ClusterManagement.List_All_Indices
224 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard
225 ... shard_name=${shard_name}
226 ... shard_type=${shard_type}
227 ... member_index_list=${all_indices}
228 ... verify_restconf=False
229 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
230 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
234 ... ${producing_transactions_time}
235 ... ${TRANSACTION_RATE_1K}
236 ... chained_flag=${CHAINED_TX}
237 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
238 ${date_start} = DateTime.Get_Current_Date
239 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
240 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${leader}
241 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
242 ${li_isolated} = BuiltIn.Set_Variable ${True}
243 BuiltIn.Wait_Until_Keyword_Succeeds
246 ... ClusterManagement.Verify_Shard_Leader_Elected
251 ... member_index_list=${follower_list}
252 ... verify_restconf=False
253 ${heal_date} = DateTime.Add_Time_To_Date ${date_start} ${heal_timeout}
254 ${sleep_to_heal} = Get_Seconds_To_Time ${heal_date}
255 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
256 BuiltIn.Should_Be_Equal ${resp} ${NONE} No response expected, received ${resp}
257 BuiltIn.Sleep ${sleep_to_heal}
258 KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${leader}
259 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
260 ${li_isolated} = BuiltIn.Set_Variable ${False}
261 BuiltIn.Wait_Until_Keyword_Succeeds
264 ... ShardStability.Shards_Stability_Get_Details
265 ... ${DEFAULT_SHARD_LIST}
266 BuiltIn.Wait_Until_Keyword_Succeeds
269 ... ClusterManagement.Get_Leader_And_Followers_For_Shard
272 ... verify_restconf=False
273 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
274 IF ${heal_timeout} < ${REQUEST_TIMEOUT}
275 Leader_Isolation_Heal_Within_Rt
277 Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
279 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
280 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
282 Leader_Isolation_PrefBasedShard_Test_Templ
283 [Documentation] Implements leader isolation test scenario.
284 [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
285 ${li_isolated} = BuiltIn.Set_Variable ${False}
286 ${producing_transactions_time} = BuiltIn.Set_Variable_If
287 ... ${heal_timeout}<${REQUEST_TIMEOUT}
288 ... ${${REQUEST_TIMEOUT}+60}
289 ... ${2*${REQUEST_TIMEOUT}}
290 ${all_indices} = ClusterManagement.List_All_Indices
291 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard
292 ... shard_name=${shard_name}!!
293 ... shard_type=${shard_type}
294 ... member_index_list=${all_indices}
295 ... verify_restconf=False
296 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
297 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
301 ... ${producing_transactions_time}
302 ... ${TRANSACTION_RATE_1K}
303 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
304 ${date_start} = DateTime.Get_Current_Date
305 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
306 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${leader}
307 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
308 ${li_isolated} = BuiltIn.Set_Variable ${True}
309 BuiltIn.Wait_Until_Keyword_Succeeds
312 ... ClusterManagement.Verify_Shard_Leader_Elected
317 ... member_index_list=${follower_list}
318 ... verify_restconf=False
319 ${heal_date} = DateTime.Add_Time_To_Date ${date_start} ${heal_timeout}
320 ${sleep_to_heal} = Get_Seconds_To_Time ${heal_date}
321 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
322 BuiltIn.Should_Be_Equal ${resp} ${NONE} No response expected, received ${resp}
323 BuiltIn.Sleep ${sleep_to_heal}
324 KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${leader}
325 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
326 ${li_isolated} = BuiltIn.Set_Variable ${False}
327 BuiltIn.Wait_Until_Keyword_Succeeds
330 ... ShardStability.Shards_Stability_Get_Details
331 ... ${DEFAULT_SHARD_LIST}
332 BuiltIn.Wait_Until_Keyword_Succeeds
335 ... ClusterManagement.Get_Leader_And_Followers_For_Shard
338 ... verify_restconf=False
339 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
340 IF ${heal_timeout} < ${REQUEST_TIMEOUT}
341 Leader_Isolation_Heal_Within_Rt
343 Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
345 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
346 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
348 Leader_Isolation_Heal_Within_Rt
349 [Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction
350 ... producers shoudl finish without error.
351 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
352 FOR ${resp} IN @{resp_list}
353 TemplatedRequests.Check_Status_Code ${resp}[2]
356 Module_Leader_Isolation_Heal_Default
357 [Documentation] The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
358 ... Then all write transaction producers should finish without error.
359 [Arguments] ${isolated_node} ${time_to_finish}
360 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
362 BuiltIn.Should_Not_Be_Equal
365 ... Write-transaction should have returned error from isolated leader node.
366 # TODO: check on response status code
367 ${restart_producer_node_idx_as_list} = BuiltIn.Create_List ${isolated_node}
368 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
369 ${restart_producer_node_ip_as_list} = BuiltIn.Create_List ${restart_producer_node_ip}
370 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
371 ... ${restart_producer_node_ip_as_list}
372 ... ${restart_producer_node_idx_as_list}
374 ... ${time_to_finish}
375 ... ${TRANSACTION_RATE_1K}
376 ... chained_flag=${CHAINED_TX}
377 ... reset_globals=${False}
378 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
379 FOR ${resp} IN @{resp_list}
380 TemplatedRequests.Check_Status_Code ${resp}[2]
383 Prefix_Leader_Isolation_Heal_Default
384 [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
385 ... Then all write transaction producers shoudl finish without error.
386 [Arguments] ${isolated_node} ${time_to_finish}
387 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
389 BuiltIn.Should_Not_Be_Equal
392 ... Produce-transaction should have returned error from isolated leader node.
393 # TODO: check on response status code
394 ${restart_producer_node_idx_as_list} = BuiltIn.Create_List ${isolated_node}
395 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
396 ${restart_producer_node_ip_as_list} = BuiltIn.Create_List ${restart_producer_node_ip}
397 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
398 ... ${restart_producer_node_ip_as_list}
399 ... ${restart_producer_node_idx_as_list}
401 ... ${time_to_finish}
402 ... ${TRANSACTION_RATE_1K}
403 ... reset_globals=${False}
404 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
405 FOR ${resp} IN @{resp_list}
406 TemplatedRequests.Check_Status_Code ${resp}[2]
409 Client_Isolation_Test_Templ
410 [Documentation] Implements client isolation test scenario.
411 [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
412 ${all_indices} = ClusterManagement.List_All_Indices
413 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard
414 ... shard_name=${shard_name}
415 ... shard_type=${shard_type}
416 ... member_index_list=${all_indices}
417 ... verify_restconf=False
418 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
419 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
420 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
421 ${client_node_ip_as_list} = BuiltIn.Create_List ${client_node_ip}
422 ${client_node_idx_as_list} = BuiltIn.Create_List ${client_node_dst}
423 ${start_date} = DateTime.Get_Current_Date
424 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${REQUEST_TIMEOUT}
425 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
426 ... ${client_node_ip_as_list}
427 ... ${client_node_idx_as_list}
429 ... ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}
430 ... ${TRANSACTION_RATE_1K}
431 ... chained_flag=${trans_chain_flag}
432 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
433 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${client_node_dst}
434 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
435 ${rpc_timeout} = Get_Seconds_To_Time ${abort_date}
436 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout
439 ... Ongoing_Transactions_Not_Failed_Yet
440 BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed
441 [Teardown] BuiltIn.Run Keywords KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${client_node_dst}
442 ... AND ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
443 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
444 ... AND MdsalLowlevelPy.Wait_For_Transactions
446 Client_Isolation_PrefBasedShard_Test_Templ
447 [Documentation] Implements client isolation test scenario.
448 [Arguments] ${listener_node_role} ${isolated_transactions_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
449 ${all_indices} = ClusterManagement.List_All_Indices
450 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard
451 ... shard_name=${shard_name}!!
452 ... shard_type=${shard_type}
453 ... member_index_list=${all_indices}
454 ... verify_restconf=False
455 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
456 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
457 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
458 ${client_node_ip_as_list} = BuiltIn.Create_List ${client_node_ip}
459 ${client_node_idx_as_list} = BuiltIn.Create_List ${client_node_dst}
460 ${start_date} = DateTime.Get_Current_Date
461 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${REQUEST_TIMEOUT}
462 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
463 ... ${client_node_ip_as_list}
464 ... ${client_node_idx_as_list}
466 ... ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}
467 ... ${TRANSACTION_RATE_1K}
468 ... isolated_transactions_flag=${isolated_transactions_flag}
469 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
470 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${client_node_dst}
471 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
472 ${rpc_timeout} = Get_Seconds_To_Time ${abort_date}
473 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout
476 ... Ongoing_Transactions_Not_Failed_Yet
477 BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed
478 [Teardown] BuiltIn.Run Keywords KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${client_node_dst}
479 ... AND ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
480 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
481 ... AND MdsalLowlevelPy.Wait_For_Transactions
483 Ongoing_Transactions_Not_Failed_Yet
484 [Documentation] Verify that no write-transaction rpc finished, means they are still running.
485 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
486 BuiltIn.Should_Be_Equal ${None} ${resp} ${resp} not expected.
488 Ongoing_Transactions_Failed
489 [Documentation] Verify if write-transaction failed.
490 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
491 Check_Status_Code ${resp}[2] explicit_status_codes=${TRANSACTION_FAILED}
494 [Documentation] Return number of seconds remaining to ${date_in_future}.
495 [Arguments] ${date_in_future}
496 ${date_now} = DateTime.Get_Current_Date
497 ${duration} = DateTime.Subtract_Date_From_Date ${date_in_future} ${date_now}
498 BuiltIn.Run_Keyword_And_Return BuiltIn.Convert_To_Integer ${duration}
500 Listener_Stability_Test_Templ
501 [Documentation] Implements listener stability test scenario for module-based shards.
502 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
503 ${subscribed} = BuiltIn.Set_Variable ${False}
504 ${all_indices} = ClusterManagement.List_All_Indices
505 ${idx_from} ${idx_to} ${idx_listen} = Get_Node_Indexes_For_The_ELM_Test
510 MdsalLowlevel.Subscribe_Dtcl ${idx_listen}
511 ${subscribed} = BuiltIn.Set_Variable ${True}
512 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
513 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
517 ... ${TRANSACTION_PRODUCTION_TIME}
518 ... ${TRANSACTION_RATE_1K}
519 ... chained_flag=${CHAINED_TX}
520 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
521 ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type}
522 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds
525 ... ClusterManagement.Verify_Shard_Leader_Elected
530 ... verify_restconf=False
531 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
532 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
533 FOR ${resp} IN @{resp_list}
534 TemplatedRequests.Check_Status_Code ${resp}[2]
536 ${copy_matches} = MdsalLowlevel.Unsubscribe_Dtcl ${idx_listen}
537 ${subscribed} = BuiltIn.Set_Variable ${False}
538 BuiltIn.Should_Be_True ${copy_matches}
539 [Teardown] BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Dtcl ${idx_listen}
541 Listener_Stability_PrefBasedShard_Test_Templ
542 [Documentation] Implements listener stability test scenario for prefix-based shards.
543 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
544 ${subscribed} = BuiltIn.Set_Variable ${False}
545 ${all_indices} = ClusterManagement.List_All_Indices
546 ${idx_from} ${idx_to} ${idx_listen} = Get_Node_Indexes_For_The_ELM_Test
551 MdsalLowlevel.Subscribe_Ddtl ${idx_listen}
552 ${subscribed} = BuiltIn.Set_Variable ${True}
553 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
554 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
558 ... ${TRANSACTION_PRODUCTION_TIME}
559 ... ${TRANSACTION_RATE_1K}
560 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
561 MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name}
562 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds
565 ... ClusterManagement.Verify_Shard_Leader_Elected
570 ... verify_restconf=False
571 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
572 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
573 FOR ${resp} IN @{resp_list}
574 TemplatedRequests.Check_Status_Code ${resp}[2]
576 ${copy_matches} = MdsalLowlevel.Unsubscribe_Ddtl ${idx_listen}
577 ${subscribed} = BuiltIn.Set_Variable ${False}
578 BuiltIn.Should_Be_True ${copy_matches}
579 [Teardown] BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Ddtl ${idx_listen}
581 Create_Prefix_Based_Shard_And_Verify
582 [Documentation] Create prefix based shard with replicas on all nodes
583 [Arguments] ${prefix}=${PREF_BASED_SHARD}
584 ${all_indices} = ClusterManagement.List_All_Indices
585 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
586 MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices}
587 BuiltIn.Wait_Until_Keyword_Succeeds
590 ... ClusterManagement.Get_Leader_And_Followers_For_Shard
591 ... shard_name=${prefix}!!
592 ... shard_type=${SHARD_TYPE}
593 ... member_index_list=${all_indices}
594 ... verify_restconf=False
596 Remove_Prefix_Based_Shard_And_Verify
597 [Documentation] Remove prefix based shard with all replicas
598 [Arguments] ${prefix}=${PREF_BASED_SHARD}
599 ${all_indices} = ClusterManagement.List_All_Indices
600 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
601 MdsalLowlevel.Remove_Prefix_Shard ${node_to_trigger} ${prefix}
602 FOR ${idx} IN @{all_indices}
603 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s Verify_Shard_Replica_Not_Present ${idx} ${prefix}!!
607 Verify_Shard_Replica_Not_Present
608 [Documentation] Verify that shard is removed. Jolokia return 404 for shard memeber.
609 [Arguments] ${member_index} ${shard_name} ${shard_type}
610 ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
611 ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type}
612 ${uri} = BuiltIn.Set_Variable
613 ... /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
614 ${text} = TemplatedRequests.Get_From_Uri uri=${uri} session=${session}
615 BuiltIn.Should_Contain ${text} "status":404 javax.management.InstanceNotFoundException
618 [Documentation] Stop every odl node and start again.
619 ClusterManagement.Stop_Members_From_List_Or_All
620 ClusterManagement.Clean_Directories_On_List_Or_All tmp_dir=/tmp
621 ClusterManagement.Start_Members_From_List_Or_All
622 BuiltIn.Wait_Until_Keyword_Succeeds
625 ... ShardStability.Shards_Stability_Get_Details
626 ... ${DEFAULT_SHARD_LIST}
627 ... verify_restconf=True
628 ClusterManagement.Run_Bash_Command_On_List_Or_All ps -ef | grep java
630 Check_Status_Of_First_Response
631 [Documentation] Extract first item from the list, third item of the tuple and call TemplatedRequests to check the http status code.
632 [Arguments] ${resp_list}
633 # ${resp_list}[0][2] does not work
634 ${tuple} = BuiltIn.Set_Variable ${resp_list}[0]
635 TemplatedRequests.Check_Status_Code ${tuple}[2]
637 Change_Use_Tell_Based_Protocol
638 [Documentation] Change status use-tell-based-protocol to True or False
639 [Arguments] ${status} ${DATASTORE_CFG}
640 ClusterManagement.Check_Bash_Command_On_List_Or_All
641 ... sed -ie "s/^#use-tell-based-protocol=true/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
642 ClusterManagement.Check_Bash_Command_On_List_Or_All
643 ... sed -ie "s/^#use-tell-based-protocol=false/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
644 BuiltIn.Run_Keyword_And_Return_If
645 ... "${status}" == "True"
646 ... ClusterManagement.Check_Bash_Command_On_List_Or_All
647 ... sed -ie "s/^use-tell-based-protocol=false/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
648 BuiltIn.Run_Keyword_And_Return_If
649 ... "${status}" == "False"
650 ... ClusterManagement.Check_Bash_Command_On_List_Or_All
651 ... sed -ie "s/^use-tell-based-protocol=true/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
652 BuiltIn.Fail Failure in status. Status can be True or False.