2 Documentation DOMDataBroker testing: Common keywords
4 ... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
6 ... This program and the accompanying materials are made available under the
7 ... terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 ... and is available at http://www.eclipse.org/legal/epl-v10.html
10 ... This resource file implements various test cases templates.
11 ... FIXME: add a link to a document (when published) where the scenarios are defined
12 Library ${CURDIR}/../MdsalLowlevelPy.py
13 Resource ${CURDIR}/../ClusterAdmin.robot
14 Resource ${CURDIR}/../ClusterManagement.robot
15 Resource ${CURDIR}/../MdsalLowlevel.robot
16 Resource ${CURDIR}/../TemplatedRequests.robot
17 Resource ${CURDIR}/../ShardStability.robot
18 Resource ${CURDIR}/../WaitForFailure.robot
23 ${TRANSACTION_RATE_1K} ${1000}
27 ${TRANSACTION_TIMEOUT} ${30}
28 ${TRANSACTION_TIMEOUT_2X} ${2*${TRANSACTION_TIMEOUT}}
32 @{TRANSACTION_FAILED} ${500}
33 ${PREF_BASED_SHARD} id-ints
34 ${TEST_LOG_LEVEL} info
35 @{TEST_LOG_COMPONENTS} org.opendaylight.controller.cluster.sharding org.opendaylight.controller.cluster.datastore
38 Explicit_Leader_Movement_Test_Templ
39 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
40 [Documentation] Implements explicit leader movement test scenario.
41 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}
43 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
44 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
45 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
46 ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type}
47 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}
48 ... ${shard_type} ${True} ${idx_from}
49 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
50 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
51 TemplatedRequests.Check_Status_Code @{resp_list}[0]
53 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
54 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
55 [Documentation] Implements explicit leader movement test scenario.
56 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}!!
58 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
59 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
60 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
61 MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name} ${ID_PREFIX}
62 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!!
63 ... ${shard_type} ${True} ${idx_from}
64 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
65 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
66 TemplatedRequests.Check_Status_Code @{resp_list}[0]
68 Get_Node_Indexes_For_The_ELM_Test
69 [Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type}
70 [Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
71 ... producer should be deployed.
72 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
73 ${idx_from} = BuiltIn.Set_Variable ${leader}
74 ${idx_to} = BuiltIn.Set_Variable @{follower_list}[0]
75 ${idx_trans} = BuiltIn.Set_Variable_If "${leader_from}" == "remote" and "${leader_to}" == "remote" @{follower_list}[1] "${leader_from}" == "local" ${leader} "${leader_to}" == "local"
76 ... @{follower_list}[0]
77 BuiltIn.Return_From_Keyword ${idx_from} ${idx_to} ${idx_trans}
79 Clean_Leader_Shutdown_Test_Templ
80 [Arguments] ${leader_location} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
81 [Documentation] Implements clean leader shutdown test scenario.
82 ${removed} = BuiltIn.Set_Variable ${False}
83 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name} ${shard_type}
84 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
85 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
86 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
87 ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
88 ${removed} = BuiltIn.Set_Variable ${True}
89 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
90 TemplatedRequests.Check_Status_Code @{resp_list}[0]
91 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard
92 ... shard_name=${shard_name} shard_type=${shard_type} member_index_list=${follower_list}
93 ... AND ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
94 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
95 ... shard_type=${shard_type}
97 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
98 [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
99 [Documentation] Implements clean leader shutdown test scenario.
100 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name}!! ${shard_type}
101 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
102 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
103 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
104 ClusterAdmin.Remove_Prefix_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
105 BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
106 BuiltIn.Run_Keyword_And_Ignore_Error ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
107 WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 10s 2s ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
108 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
109 TemplatedRequests.Check_Status_Code @{resp_list}[0]
110 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard
111 ... shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
112 ... AND ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
113 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
114 ... shard_type=${shard_type}
116 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
117 [Arguments] ${leader_location} ${shard_name} ${shard_type}
118 [Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
119 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
120 ${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1
121 BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth}
122 ${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} @{follower_list}[0]
123 BuiltIn.Return_From_Keyword ${producer_idx} ${leader} ${follower_list}
125 Leader_Isolation_Test_Templ
126 [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
127 [Documentation] Implements leader isolation test scenario.
128 ${li_isolated} BuiltIn.Set_Variable ${False}
129 ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
130 ${all_indices} = ClusterManagement.List_All_Indices
131 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
132 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
133 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
134 ${date_start} = DateTime.Get_Current_Date
135 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
136 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
137 ${li_isolated} BuiltIn.Set_Variable ${True}
138 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True}
139 ... ${leader} member_index_list=${follower_list}
140 BuiltIn.Sleep ${heal_timeout}
141 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
142 ${li_isolated} BuiltIn.Set_Variable ${False}
143 BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
144 BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type}
145 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
146 BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
147 ... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
148 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
149 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
151 Leader_Isolation_PrefBasedShard_Test_Templ
152 [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
153 [Documentation] Implements leader isolation test scenario.
154 ${li_isolated} BuiltIn.Set_Variable ${False}
155 ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
156 ${all_indices} = ClusterManagement.List_All_Indices
157 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
158 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
159 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K}
160 ${date_start} = DateTime.Get_Current_Date
161 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
162 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
163 ${li_isolated} BuiltIn.Set_Variable ${True}
164 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True}
165 ... ${leader} member_index_list=${follower_list}
166 BuiltIn.Sleep ${heal_timeout}
167 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
168 ${li_isolated} BuiltIn.Set_Variable ${False}
169 BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
170 BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type}
171 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
172 BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
173 ... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
174 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
175 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
177 Leader_Isolation_Heal_Within_Tt
178 [Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction
179 ... producers shoudl finish without error.
180 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
181 : FOR ${resp} IN @{resp_list}
182 \ TemplatedRequests.Check_Status_Code ${resp}
184 Module_Leader_Isolation_Heal_Default
185 [Arguments] ${isolated_node} ${time_to_finish}
186 [Documentation] The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
187 ... Then all write transaction producers should finish without error.
188 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
190 # TODO: check on response status code
191 ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
192 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
193 ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
194 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
195 ... reset_globals=${False}
196 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
197 : FOR ${resp} IN @{resp_list}
198 \ TemplatedRequests.Check_Status_Code ${resp}
200 Prefix_Leader_Isolation_Heal_Default
201 [Arguments] ${isolated_node} ${time_to_finish}
202 [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
203 Then all write transaction producers shoudl finish without error.
204 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
206 # TODO: check on response status code
207 ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
208 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
209 ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
210 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} reset_globals=${False}
211 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
212 : FOR ${resp} IN @{resp_list}
213 \ TemplatedRequests.Check_Status_Code ${resp}
215 Client_Isolation_Test_Templ
216 [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
217 [Documentation] Implements client isolation test scenario.
218 ${all_indices} = ClusterManagement.List_All_Indices
219 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
220 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
221 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
222 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
223 ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
224 ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
225 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag}
226 ${start_date} DateTime.Get_Current_Date
227 ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT}
228 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT}
229 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
230 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet
231 WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed
232 ${abort_time} Get_Seconds_To_Time ${abort_date}
233 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False}
234 WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True}
235 [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
236 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
237 ... AND MdsalLowlevelPy.Wait_For_Transactions
239 Client_Isolation_PrefBasedShard_Test_Templ
240 [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
241 [Documentation] Implements client isolation test scenario.
242 ${all_indices} = ClusterManagement.List_All_Indices
243 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
244 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
245 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
246 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
247 ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
248 ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
249 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
250 ${start_date} DateTime.Get_Current_Date
251 ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT}
252 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT}
253 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
254 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet
255 WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed
256 ${abort_time} Get_Seconds_To_Time ${abort_date}
257 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False}
258 WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True}
259 [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
260 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
261 ... AND MdsalLowlevelPy.Wait_For_Transactions
263 Ongoing_Transactions_Not_Failed_Yet
264 [Documentation] Verify that no write-transaction rpc finished, means they are still running.
265 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
266 BuiltIn.Should_Be_Equal ${None} ${resp} ${resp} not expected.
268 Ongoing_Transactions_Failed
269 [Documentation] Verify if write-transaction failed.
270 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
271 Check_Status_Code ${resp} explicit_status_codes=${TRANSACTION_FAILED}
274 [Arguments] ${date_in_future}
275 [Documentation] Return number of seconds remaining to ${date_in_future}.
276 ${date_now} = DateTime.Get_Current_Date
277 ${duration} = DateTime.Subtract_Date_From_Date ${date_in_future} ${date_now}
278 BuiltIn.Run_Keyword_And_Return BuiltIn.Convert_To_Integer ${duration}
280 Remote_Listener_Test_Templ
281 [Arguments] ${listener_node_role} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
282 [Documentation] Implements remote listener test scenario.
283 ${all_indices} = ClusterManagement.List_All_Indices
284 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
285 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
286 ${follower2} = Collections.Get_From_List ${follower_list} ${1}
287 ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
288 MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst}
289 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
290 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
291 ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type}
292 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True}
294 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
295 : FOR ${resp} IN @{resp_list}
296 \ TemplatedRequests.Check_Status_Code ${resp}
297 [Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
299 Remote_Listener_PrefBasedShard_Test_Templ
300 [Arguments] ${listener_node_role} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
301 [Documentation] Implements listener isolation test scenario.
302 ${all_indices} = ClusterManagement.List_All_Indices
303 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
304 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
305 ${follower2} = Collections.Get_From_List ${follower_list} ${1}
306 ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
307 MdsalLowlevel.Subscribe_Ddtl ${listener_node_dst}
308 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
309 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K}
310 MdsalLowlevel.Become_Prefix_Leader ${follower1} ${shard_name} ${ID_PREFIX}
311 BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True}
313 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
314 : FOR ${resp} IN @{resp_list}
315 \ TemplatedRequests.Check_Status_Code ${resp}
316 [Teardown] MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst}
318 Create_Prefix_Based_Shard_And_Verify
319 [Arguments] ${prefix}=${PREF_BASED_SHARD}
320 [Documentation] Create prefix based shard with replicas on all nodes
321 ${all_indices} = ClusterManagement.List_All_Indices
322 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
323 MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices}
324 BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${prefix}!! shard_type=${SHARD_TYPE} member_index_list=${all_indices}
326 Remove_Prefix_Based_Shard_And_Verify
327 [Arguments] ${prefix}=${PREF_BASED_SHARD}
328 [Documentation] Remove prefix based shard with all replicas
329 ${all_indices} = ClusterManagement.List_All_Indices
330 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
331 MdsalLowlevel.Remove_Prefix_Shard ${node_to_trigger} ${prefix}
332 : FOR ${idx} IN @{all_indices}
333 \ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s Verify_Shard_Replica_Removed ${idx} ${prefix}!!
336 Verify_Shard_Replica_Removed
337 [Arguments] ${member_index} ${shard_name} ${shard_type}
338 [Documentation] Verify that shard is removed. Jolokia return 404 for shard memeber.
339 ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
340 ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type}
341 ${uri} = BuiltIn.Set_Variable /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
342 ${text} TemplatedRequests.Get_From_Uri uri=${uri} session=${session}
343 BuiltIn.Should_Contain ${text} "status":404 javax.management.InstanceNotFoundException