2 Documentation DOMDataBroker testing: Common keywords
4 ... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
6 ... This program and the accompanying materials are made available under the
7 ... terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 ... and is available at http://www.eclipse.org/legal/epl-v10.html
10 ... This resource file implements various test cases templates.
11 ... FIXME: add a link to a document (when published) where the scenarios are defined
12 Library ${CURDIR}/../MdsalLowlevelPy.py
13 Resource ${CURDIR}/../ClusterAdmin.robot
14 Resource ${CURDIR}/../ClusterManagement.robot
15 Resource ${CURDIR}/../MdsalLowlevel.robot
16 Resource ${CURDIR}/../TemplatedRequests.robot
17 Resource ${CURDIR}/../ShardStability.robot
18 Resource ${CURDIR}/../WaitForFailure.robot
23 ${TRANSACTION_RATE_1K} ${1000}
24 ${DURATION_90S} ${90} #TODO: Rename to hint at the goal of waiting, instead of its default value.
28 ${TRANSACTION_TIMEOUT} ${30}
29 ${TRANSACTION_TIMEOUT_2X} ${2*${TRANSACTION_TIMEOUT}}
32 ${ISOLATED_TRANS_TRUE} ${True}
33 ${ISOLATED_TRANS_FALSE} ${False}
34 ${HARD_TIMEOUT} ${2*${TRANSACTION_TIMEOUT}}
35 @{TRANSACTION_FAILED} ${500}
36 ${PREF_BASED_SHARD} id-ints
37 ${TEST_LOG_LEVEL} info
38 @{TEST_LOG_COMPONENTS} org.opendaylight.controller.cluster.sharding org.opendaylight.controller.cluster.datastore
41 Explicit_Leader_Movement_Test_Templ
42 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
43 [Documentation] Implements explicit leader movement test scenario.
44 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}
46 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
47 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
48 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
49 ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type}
50 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}
51 ... ${shard_type} ${True} ${idx_from}
52 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
53 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
54 TemplatedRequests.Check_Status_Code @{resp_list}[0]
56 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
57 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
58 [Documentation] Implements explicit leader movement test scenario.
59 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}!!
61 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
62 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
63 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
64 MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name} ${ID_PREFIX}
65 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!!
66 ... ${shard_type} ${True} ${idx_from}
67 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
68 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
69 TemplatedRequests.Check_Status_Code @{resp_list}[0]
71 Get_Node_Indexes_For_The_ELM_Test
72 [Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type}
73 [Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
74 ... producer should be deployed.
75 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
76 ${idx_from} = BuiltIn.Set_Variable ${leader}
77 ${idx_to} = BuiltIn.Set_Variable @{follower_list}[0]
78 ${idx_trans} = BuiltIn.Set_Variable_If "${leader_from}" == "remote" and "${leader_to}" == "remote" @{follower_list}[1] "${leader_from}" == "local" ${leader} "${leader_to}" == "local"
79 ... @{follower_list}[0]
80 BuiltIn.Return_From_Keyword ${idx_from} ${idx_to} ${idx_trans}
82 Clean_Leader_Shutdown_Test_Templ
83 [Arguments] ${leader_location} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
84 [Documentation] Implements clean leader shutdown test scenario.
85 ${removed} = BuiltIn.Set_Variable ${False}
86 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name} ${shard_type}
87 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
88 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
89 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
90 ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
91 ${removed} = BuiltIn.Set_Variable ${True}
92 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
93 TemplatedRequests.Check_Status_Code @{resp_list}[0]
94 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard
95 ... shard_name=${shard_name} shard_type=${shard_type} member_index_list=${follower_list}
96 ... AND ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
97 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
98 ... shard_type=${shard_type}
100 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
101 [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
102 [Documentation] Implements clean leader shutdown test scenario.
103 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name}!! ${shard_type}
104 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
105 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
106 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
107 ClusterAdmin.Remove_Prefix_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
108 BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
109 BuiltIn.Run_Keyword_And_Ignore_Error ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
110 WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 10s 2s ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
111 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
112 TemplatedRequests.Check_Status_Code @{resp_list}[0]
113 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard
114 ... shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
115 ... AND ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
116 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
117 ... shard_type=${shard_type}
119 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
120 [Arguments] ${leader_location} ${shard_name} ${shard_type}
121 [Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
122 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
123 ${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1
124 BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth}
125 ${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} @{follower_list}[0]
126 BuiltIn.Return_From_Keyword ${producer_idx} ${leader} ${follower_list}
128 Leader_Isolation_Test_Templ
129 [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
130 [Documentation] Implements leader isolation test scenario.
131 ${li_isolated} BuiltIn.Set_Variable ${False}
132 ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
133 ${all_indices} = ClusterManagement.List_All_Indices
134 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
135 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
136 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
137 ${date_start} = DateTime.Get_Current_Date
138 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
139 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
140 ${li_isolated} BuiltIn.Set_Variable ${True}
141 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True}
142 ... ${leader} member_index_list=${follower_list}
143 BuiltIn.Sleep ${heal_timeout}
144 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
145 ${li_isolated} BuiltIn.Set_Variable ${False}
146 BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
147 BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type}
148 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
149 BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
150 ... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
151 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
152 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
154 Leader_Isolation_PrefBasedShard_Test_Templ
155 [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
156 [Documentation] Implements leader isolation test scenario.
157 ${li_isolated} BuiltIn.Set_Variable ${False}
158 ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
159 ${all_indices} = ClusterManagement.List_All_Indices
160 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
161 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
162 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K}
163 ${date_start} = DateTime.Get_Current_Date
164 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
165 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
166 ${li_isolated} BuiltIn.Set_Variable ${True}
167 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True}
168 ... ${leader} member_index_list=${follower_list}
169 BuiltIn.Sleep ${heal_timeout}
170 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
171 ${li_isolated} BuiltIn.Set_Variable ${False}
172 BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
173 BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type}
174 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
175 BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
176 ... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
177 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
178 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
180 Leader_Isolation_Heal_Within_Tt
181 [Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction
182 ... producers shoudl finish without error.
183 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
184 : FOR ${resp} IN @{resp_list}
185 \ TemplatedRequests.Check_Status_Code ${resp}
187 Module_Leader_Isolation_Heal_Default
188 [Arguments] ${isolated_node} ${time_to_finish}
189 [Documentation] The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
190 ... Then all write transaction producers should finish without error.
191 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
193 # TODO: check on response status code
194 ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
195 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
196 ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
197 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
198 ... reset_globals=${False}
199 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
200 : FOR ${resp} IN @{resp_list}
201 \ TemplatedRequests.Check_Status_Code ${resp}
203 Prefix_Leader_Isolation_Heal_Default
204 [Arguments] ${isolated_node} ${time_to_finish}
205 [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
206 Then all write transaction producers shoudl finish without error.
207 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
209 # TODO: check on response status code
210 ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
211 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
212 ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
213 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} reset_globals=${False}
214 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
215 : FOR ${resp} IN @{resp_list}
216 \ TemplatedRequests.Check_Status_Code ${resp}
218 Client_Isolation_Test_Templ
219 [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
220 [Documentation] Implements client isolation test scenario.
221 ${all_indices} = ClusterManagement.List_All_Indices
222 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
223 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
224 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
225 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
226 ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
227 ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
228 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_90S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag}
229 ${start_date} DateTime.Get_Current_Date
230 ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT}
231 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT}
232 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
233 ${trans_timeout} = Get_Seconds_To_Time ${timeout_date}
234 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${trans_timeout} 1s Ongoing_Transactions_Not_Failed_Yet
235 ${hard_timeout} = Get_Seconds_To_Time ${abort_date}
236 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${hard_timeout} 1s Ongoing_Transactions_Not_Failed_Yet
237 BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed
238 [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
239 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
240 ... AND MdsalLowlevelPy.Wait_For_Transactions
242 Client_Isolation_PrefBasedShard_Test_Templ
243 [Arguments] ${listener_node_role} ${isolated_transactions_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
244 [Documentation] Implements client isolation test scenario.
245 ${all_indices} = ClusterManagement.List_All_Indices
246 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
247 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
248 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
249 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
250 ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
251 ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
252 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_90S} ${TRANSACTION_RATE_1K} isolated_transactions_flag=${isolated_transactions_flag}
253 ${start_date} DateTime.Get_Current_Date
254 ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT}
255 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT}
256 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
257 ${trans_timeout} = Get_Seconds_To_Time ${timeout_date}
258 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${trans_timeout} 1s Ongoing_Transactions_Not_Failed_Yet
259 ${hard_timeout} = Get_Seconds_To_Time ${abort_date}
260 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${hard_timeout} 1s Ongoing_Transactions_Not_Failed_Yet
261 BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed
262 [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
263 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
264 ... AND MdsalLowlevelPy.Wait_For_Transactions
266 Ongoing_Transactions_Not_Failed_Yet
267 [Documentation] Verify that no write-transaction rpc finished, means they are still running.
268 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
269 BuiltIn.Should_Be_Equal ${None} ${resp} ${resp} not expected.
271 Ongoing_Transactions_Failed
272 [Documentation] Verify if write-transaction failed.
273 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
274 Check_Status_Code ${resp} explicit_status_codes=${TRANSACTION_FAILED}
277 [Arguments] ${date_in_future}
278 [Documentation] Return number of seconds remaining to ${date_in_future}.
279 ${date_now} = DateTime.Get_Current_Date
280 ${duration} = DateTime.Subtract_Date_From_Date ${date_in_future} ${date_now}
281 BuiltIn.Run_Keyword_And_Return BuiltIn.Convert_To_Integer ${duration}
283 Remote_Listener_Test_Templ
284 [Arguments] ${listener_node_role} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
285 [Documentation] Implements remote listener test scenario.
286 ${subscribed} = BuiltIn.Set_Variable ${False}
287 ${all_indices} = ClusterManagement.List_All_Indices
288 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
289 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
290 ${follower2} = Collections.Get_From_List ${follower_list} ${1}
291 ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
292 MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst}
293 ${subscribed} = BuiltIn.Set_Variable ${True}
294 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
295 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
296 ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type}
297 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True}
299 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
300 : FOR ${resp} IN @{resp_list}
301 \ TemplatedRequests.Check_Status_Code ${resp}
302 ${copy_matches} = MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
303 ${subscribed} = BuiltIn.Set_Variable ${False}
304 BuiltIn.Should_Be_True ${copy_matches}
305 [Teardown] BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
307 Remote_Listener_PrefBasedShard_Test_Templ
308 [Arguments] ${listener_node_role} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
309 [Documentation] Implements listener isolation test scenario.
310 ${subscribed} = BuiltIn.Set_Variable ${False}
311 ${all_indices} = ClusterManagement.List_All_Indices
312 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
313 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
314 ${follower2} = Collections.Get_From_List ${follower_list} ${1}
315 ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
316 MdsalLowlevel.Subscribe_Ddtl ${listener_node_dst}
317 ${subscribed} = BuiltIn.Set_Variable ${True}
318 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
319 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K}
320 MdsalLowlevel.Become_Prefix_Leader ${follower1} ${shard_name} ${ID_PREFIX}
321 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True}
323 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
324 : FOR ${resp} IN @{resp_list}
325 \ TemplatedRequests.Check_Status_Code ${resp}
326 ${copy_matches} = MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst}
327 ${subscribed} = BuiltIn.Set_Variable ${False}
328 BuiltIn.Should_Be_True ${copy_matches}
329 [Teardown] BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst}
331 Create_Prefix_Based_Shard_And_Verify
332 [Arguments] ${prefix}=${PREF_BASED_SHARD}
333 [Documentation] Create prefix based shard with replicas on all nodes
334 ${all_indices} = ClusterManagement.List_All_Indices
335 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
336 MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices}
337 BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${prefix}!! shard_type=${SHARD_TYPE} member_index_list=${all_indices}
339 Remove_Prefix_Based_Shard_And_Verify
340 [Arguments] ${prefix}=${PREF_BASED_SHARD}
341 [Documentation] Remove prefix based shard with all replicas
342 ${all_indices} = ClusterManagement.List_All_Indices
343 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
344 MdsalLowlevel.Remove_Prefix_Shard ${node_to_trigger} ${prefix}
345 : FOR ${idx} IN @{all_indices}
346 \ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s Verify_Shard_Replica_Removed ${idx} ${prefix}!!
349 Verify_Shard_Replica_Removed
350 [Arguments] ${member_index} ${shard_name} ${shard_type}
351 [Documentation] Verify that shard is removed. Jolokia return 404 for shard memeber.
352 ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
353 ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type}
354 ${uri} = BuiltIn.Set_Variable /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
355 ${text} TemplatedRequests.Get_From_Uri uri=${uri} session=${session}
356 BuiltIn.Should_Contain ${text} "status":404 javax.management.InstanceNotFoundException