2 Documentation DOMDataBroker testing: Common keywords
4 ... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
6 ... This program and the accompanying materials are made available under the
7 ... terms of the Eclipse Public License v1.0 which accompanies this distribution,
8 ... and is available at http://www.eclipse.org/legal/epl-v10.html
10 ... This resource file implements various test cases templates.
11 ... FIXME: add a link to a document (when published) where the scenarios are defined
13 ... TODO: When checking first response in isolation scenarior, make sure it comes from the expected member.
14 Library ${CURDIR}/../MdsalLowlevelPy.py
15 Resource ${CURDIR}/../ClusterAdmin.robot
16 Resource ${CURDIR}/../ClusterManagement.robot
17 Resource ${CURDIR}/../KarafKeywords.robot
18 Resource ${CURDIR}/../MdsalLowlevel.robot
19 Resource ${CURDIR}/../TemplatedRequests.robot
20 Resource ${CURDIR}/../ShardStability.robot
21 Resource ${CURDIR}/../WaitForFailure.robot
26 ${TRANSACTION_RATE_1K} ${1000}
27 ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT} ${2*${REQUEST_TIMEOUT}}
28 ${TRANSACTION_PRODUCTION_TIME} ${40}
29 ${SLEEP_AFTER_TRANSACTIONS_INIT} 5s
31 ${ID_PREFIX2} different-prefix-
34 ${ISOLATED_TRANS_TRUE} ${True}
35 ${ISOLATED_TRANS_FALSE} ${False}
36 ${JAVA_INTERNAL_RECONNECT_TIMEOUT} ${30}
37 ${REQUEST_TIMEOUT} ${120}
38 ${HEAL_WITHIN_REQUEST_TIMEOUT} ${${JAVA_INTERNAL_RECONNECT_TIMEOUT}+10}
39 ${HEAL_AFTER_REQUEST_TIMEOUT} ${${REQUEST_TIMEOUT}+10}
40 @{TRANSACTION_FAILED} ${500}
41 ${PREF_BASED_SHARD} id-ints
42 ${TEST_LOG_LEVEL} info
43 @{TEST_LOG_COMPONENTS} org.opendaylight.controller.cluster.sharding org.opendaylight.controller.cluster.datastore
44 ${HEAL_WITHIN_TRANS_TIMEOUT} ${0}
47 Explicit_Leader_Movement_Test_Templ
48 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
49 [Documentation] Implements explicit leader movement test scenario.
50 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}
52 KarafKeywords.Log_Message_To_Controller_Karaf Starting leader movement from node${idx_from} to node${idx_to}, transaction producer at node${idx_trans}.
53 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
54 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
55 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
56 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
57 ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type}
58 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}
59 ... ${shard_type} ${True} ${idx_from} verify_restconf=False
60 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
61 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
62 Check_Status_Of_First_Response ${resp_list}
64 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
65 [Arguments] ${leader_from} ${leader_to} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
66 [Documentation] Implements explicit leader movement test scenario.
67 ${idx_from} ${idx_to} ${idx_trans} = Get_Node_Indexes_For_The_ELM_Test ${leader_from} ${leader_to} ${shard_name}!!
69 ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP}
70 ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans}
71 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME} ${TRANSACTION_RATE_1K}
72 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
73 MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name}
74 ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!!
75 ... ${shard_type} ${True} ${idx_from} verify_restconf=False
76 BuiltIn.Should_Be_Equal ${idx_to} ${new_leader}
77 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
78 Check_Status_Of_First_Response ${resp_list}
80 Get_Node_Indexes_For_The_ELM_Test
81 [Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type}
82 [Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
83 ... producer should be deployed.
84 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} verify_restconf=False
85 ${idx_from} = BuiltIn.Set_Variable ${leader}
86 ${idx_to} = BuiltIn.Set_Variable @{follower_list}[0]
87 ${idx_trans} = BuiltIn.Set_Variable_If "${leader_from}" == "remote" and "${leader_to}" == "remote" @{follower_list}[1] "${leader_from}" == "local" ${leader} "${leader_to}" == "local"
88 ... @{follower_list}[0]
89 BuiltIn.Return_From_Keyword ${idx_from} ${idx_to} ${idx_trans}
91 Clean_Leader_Shutdown_Test_Templ
92 [Arguments] ${leader_location} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
93 [Documentation] Implements clean leader shutdown test scenario.
94 ${removed} = BuiltIn.Set_Variable ${False}
95 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name} ${shard_type}
96 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
97 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
98 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
99 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
100 ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
101 ${removed} = BuiltIn.Set_Variable ${True}
102 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
103 Check_Status_Of_First_Response ${resp_list}
104 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard
105 ... shard_name=${shard_name} shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
106 ... AND ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
107 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
108 ... verify_restconf=False shard_type=${shard_type}
110 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
111 [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
112 [Documentation] Implements clean leader shutdown test scenario.
113 ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name}!! ${shard_type}
114 ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP}
115 ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx}
116 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME} ${TRANSACTION_RATE_1K}
117 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
118 ClusterAdmin.Remove_Prefix_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
119 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
120 ... verify_restconf=False
121 # TODO: Check on the result of this
122 BuiltIn.Run_Keyword_And_Ignore_Error ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
123 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
124 Check_Status_Of_First_Response ${resp_list}
125 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard
126 ... shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
127 ... AND ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
128 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
129 ... verify_restconf=False shard_type=${shard_type}
131 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
132 [Arguments] ${leader_location} ${shard_name} ${shard_type}
133 [Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
134 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} verify_restconf=False
135 ${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1
136 BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth}
137 ${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} @{follower_list}[0]
138 BuiltIn.Return_From_Keyword ${producer_idx} ${leader} ${follower_list}
140 Leader_Isolation_Test_Templ
141 [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
142 [Documentation] Implements leader isolation test scenario.
143 ${li_isolated} BuiltIn.Set_Variable ${False}
144 ${producing_transactions_time} = BuiltIn.Set_Variable_If ${heal_timeout}<${REQUEST_TIMEOUT} ${${REQUEST_TIMEOUT}+60} ${2*${REQUEST_TIMEOUT}}
145 ${all_indices} = ClusterManagement.List_All_Indices
146 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
147 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
148 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
149 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
150 ${date_start} = DateTime.Get_Current_Date
151 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
152 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${leader}
153 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
154 ${li_isolated} BuiltIn.Set_Variable ${True}
155 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True}
156 ... ${leader} member_index_list=${follower_list}
157 ${heal_date} = DateTime.Add_Time_To_Date ${date_start} ${heal_timeout}
158 ${sleep_to_heal} = Get_Seconds_To_Time ${heal_date}
159 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
160 BuiltIn.Should_Be_Equal ${resp} ${NONE} No response expected, received ${resp}
161 BuiltIn.Sleep ${sleep_to_heal}
162 KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${leader}
163 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
164 ${li_isolated} BuiltIn.Set_Variable ${False}
165 BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
166 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type} verify_restconf=False
167 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
168 BuiltIn.Run_Keyword_If ${heal_timeout} < ${REQUEST_TIMEOUT} Leader_Isolation_Heal_Within_Rt
169 ... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
170 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
171 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
173 Leader_Isolation_PrefBasedShard_Test_Templ
174 [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
175 [Documentation] Implements leader isolation test scenario.
176 ${li_isolated} BuiltIn.Set_Variable ${False}
177 ${producing_transactions_time} = BuiltIn.Set_Variable_If ${heal_timeout}<${REQUEST_TIMEOUT} ${${REQUEST_TIMEOUT}+60} ${2*${REQUEST_TIMEOUT}}
178 ${all_indices} = ClusterManagement.List_All_Indices
179 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
180 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
181 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K}
182 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
183 ${date_start} = DateTime.Get_Current_Date
184 ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time}
185 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${leader}
186 ClusterManagement.Isolate_Member_From_List_Or_All ${leader}
187 ${li_isolated} BuiltIn.Set_Variable ${True}
188 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True}
189 ... ${leader} member_index_list=${follower_list}
190 ${heal_date} = DateTime.Add_Time_To_Date ${date_start} ${heal_timeout}
191 ${sleep_to_heal} = Get_Seconds_To_Time ${heal_date}
192 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
193 BuiltIn.Should_Be_Equal ${resp} ${NONE} No response expected, received ${resp}
194 BuiltIn.Sleep ${sleep_to_heal}
195 KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${leader}
196 ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
197 ${li_isolated} BuiltIn.Set_Variable ${False}
198 BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
199 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type} verify_restconf=False
200 ${time_to_finish} = Get_Seconds_To_Time ${date_end}
201 BuiltIn.Run_Keyword_If ${heal_timeout} < ${REQUEST_TIMEOUT} Leader_Isolation_Heal_Within_Rt
202 ... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
203 [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
204 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
206 Leader_Isolation_Heal_Within_Rt
207 [Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction
208 ... producers shoudl finish without error.
209 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
210 : FOR ${resp} IN @{resp_list}
211 \ TemplatedRequests.Check_Status_Code @{resp}[2]
213 Module_Leader_Isolation_Heal_Default
214 [Arguments] ${isolated_node} ${time_to_finish}
215 [Documentation] The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
216 ... Then all write transaction producers should finish without error.
217 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
219 BuiltIn.Should_Not_Be_Equal ${NONE} ${resp} Write-transaction should have returned error from isolated leader node.
220 # TODO: check on response status code
221 ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
222 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
223 ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
224 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX2} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
225 ... reset_globals=${False}
226 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
227 : FOR ${resp} IN @{resp_list}
228 \ TemplatedRequests.Check_Status_Code @{resp}[2]
230 Prefix_Leader_Isolation_Heal_Default
231 [Arguments] ${isolated_node} ${time_to_finish}
232 [Documentation] The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
233 ... Then all write transaction producers shoudl finish without error.
234 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
236 BuiltIn.Should_Not_Be_Equal ${NONE} ${resp} Produce-transaction should have returned error from isolated leader node.
237 # TODO: check on response status code
238 ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node}
239 ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node}
240 ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip}
241 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX2} ${time_to_finish} ${TRANSACTION_RATE_1K} reset_globals=${False}
242 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
243 : FOR ${resp} IN @{resp_list}
244 \ TemplatedRequests.Check_Status_Code @{resp}[2]
246 Client_Isolation_Test_Templ
247 [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
248 [Documentation] Implements client isolation test scenario.
249 ${all_indices} = ClusterManagement.List_All_Indices
250 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
251 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
252 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
253 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
254 ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
255 ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
256 ${start_date} DateTime.Get_Current_Date
257 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${REQUEST_TIMEOUT}
258 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag}
259 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
260 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${client_node_dst}
261 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
262 ${rpc_timeout} = Get_Seconds_To_Time ${abort_date}
263 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${rpc_timeout} 1s Ongoing_Transactions_Not_Failed_Yet
264 BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed
265 [Teardown] BuiltIn.Run Keywords KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${client_node_dst}
266 ... AND ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
267 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
268 ... AND MdsalLowlevelPy.Wait_For_Transactions
270 Client_Isolation_PrefBasedShard_Test_Templ
271 [Arguments] ${listener_node_role} ${isolated_transactions_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
272 [Documentation] Implements client isolation test scenario.
273 ${all_indices} = ClusterManagement.List_All_Indices
274 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
275 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
276 ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
277 ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
278 ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip}
279 ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst}
280 ${start_date} DateTime.Get_Current_Date
281 ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${REQUEST_TIMEOUT}
282 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT} ${TRANSACTION_RATE_1K} isolated_transactions_flag=${isolated_transactions_flag}
283 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
284 KarafKeywords.Log_Message_To_Controller_Karaf Isolating node ${client_node_dst}
285 ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst}
286 ${rpc_timeout} = Get_Seconds_To_Time ${abort_date}
287 WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${rpc_timeout} 1s Ongoing_Transactions_Not_Failed_Yet
288 BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed
289 [Teardown] BuiltIn.Run Keywords KarafKeywords.Log_Message_To_Controller_Karaf Rejoining node ${client_node_dst}
290 ... AND ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst}
291 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 70s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
292 ... AND MdsalLowlevelPy.Wait_For_Transactions
294 Ongoing_Transactions_Not_Failed_Yet
295 [Documentation] Verify that no write-transaction rpc finished, means they are still running.
296 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
297 BuiltIn.Should_Be_Equal ${None} ${resp} ${resp} not expected.
299 Ongoing_Transactions_Failed
300 [Documentation] Verify if write-transaction failed.
301 ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response
302 Check_Status_Code @{resp}[2] explicit_status_codes=${TRANSACTION_FAILED}
305 [Arguments] ${date_in_future}
306 [Documentation] Return number of seconds remaining to ${date_in_future}.
307 ${date_now} = DateTime.Get_Current_Date
308 ${duration} = DateTime.Subtract_Date_From_Date ${date_in_future} ${date_now}
309 BuiltIn.Run_Keyword_And_Return BuiltIn.Convert_To_Integer ${duration}
311 Remote_Listener_Test_Templ
312 [Arguments] ${listener_node_role} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
313 [Documentation] Implements remote listener test scenario.
314 ${subscribed} = BuiltIn.Set_Variable ${False}
315 ${all_indices} = ClusterManagement.List_All_Indices
316 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
317 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
318 ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
319 MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst}
320 ${subscribed} = BuiltIn.Set_Variable ${True}
321 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
322 MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
323 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
324 ClusterAdmin.Remove_Shard_Replica ${leader} ${shard_name} member-${leader} ${shard_type}
325 ${newleader} ${newfollower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
326 ... shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
327 BuiltIn.Should_Not_Be_Equal_As_Numbers ${leader} ${newleader}
328 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
329 : FOR ${resp} IN @{resp_list}
330 \ TemplatedRequests.Check_Status_Code @{resp}[2]
331 ${copy_matches} = MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
332 ${subscribed} = BuiltIn.Set_Variable ${False}
333 BuiltIn.Should_Be_True ${copy_matches}
334 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst}
335 ... AND ClusterAdmin.Add_Shard_Replica ${leader} ${shard_name} ${shard_type}
336 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
337 ... verify_restconf=False shard_type=${shard_type}
339 Remote_Listener_PrefBasedShard_Test_Templ
340 [Arguments] ${listener_node_role} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
341 [Documentation] Implements listener isolation test scenario.
342 ${subscribed} = BuiltIn.Set_Variable ${False}
343 ${all_indices} = ClusterManagement.List_All_Indices
344 ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
345 ${follower1} = Collections.Get_From_List ${follower_list} ${0}
346 ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
347 MdsalLowlevel.Subscribe_Ddtl ${listener_node_dst}
348 ${subscribed} = BuiltIn.Set_Variable ${True}
349 ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
350 MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${TRANSACTION_PRODUCTION_TIME} ${TRANSACTION_RATE_1K}
351 BuiltIn.Sleep ${SLEEP_AFTER_TRANSACTIONS_INIT}
352 ClusterAdmin.Remove_Prefix_Shard_Replica ${leader} ${shard_name} member-${leader} ${shard_type}
353 ${newleader} ${newfollower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
354 ... member_index_list=${follower_list} verify_restconf=False shard_type=${shard_type}
355 BuiltIn.Should_Not_Be_Equal_As_Numbers ${leader} ${newleader}
356 ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
357 : FOR ${resp} IN @{resp_list}
358 \ TemplatedRequests.Check_Status_Code @{resp}[2]
359 ${copy_matches} = MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst}
360 ${subscribed} = BuiltIn.Set_Variable ${False}
361 BuiltIn.Should_Be_True ${copy_matches}
362 [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst}
363 ... AND ClusterAdmin.Add_Prefix_Shard_Replica ${leader} ${shard_name} ${shard_type}
364 ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
365 ... verify_restconf=False shard_type=${shard_type}
367 Create_Prefix_Based_Shard_And_Verify
368 [Arguments] ${prefix}=${PREF_BASED_SHARD}
369 [Documentation] Create prefix based shard with replicas on all nodes
370 ${all_indices} = ClusterManagement.List_All_Indices
371 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
372 MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices}
373 BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${prefix}!! shard_type=${SHARD_TYPE} member_index_list=${all_indices}
374 ... verify_restconf=False
376 Remove_Prefix_Based_Shard_And_Verify
377 [Arguments] ${prefix}=${PREF_BASED_SHARD}
378 [Documentation] Remove prefix based shard with all replicas
379 ${all_indices} = ClusterManagement.List_All_Indices
380 ${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
381 MdsalLowlevel.Remove_Prefix_Shard ${node_to_trigger} ${prefix}
382 : FOR ${idx} IN @{all_indices}
383 \ BuiltIn.Wait_Until_Keyword_Succeeds 60s 5s Verify_Shard_Replica_Removed ${idx} ${prefix}!!
386 Verify_Shard_Replica_Removed
387 [Arguments] ${member_index} ${shard_name} ${shard_type}
388 [Documentation] Verify that shard is removed. Jolokia return 404 for shard memeber.
389 ${session} = Resolve_Http_Session_For_Member member_index=${member_index}
390 ${type_class} = Resolve_Shard_Type_Class shard_type=${shard_type}
391 ${uri} = BuiltIn.Set_Variable /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
392 ${text} TemplatedRequests.Get_From_Uri uri=${uri} session=${session}
393 BuiltIn.Should_Contain ${text} "status":404 javax.management.InstanceNotFoundException
396 [Documentation] Kill every odl node and start again.
397 ClusterManagement.Kill_Members_From_List_Or_All
398 ClusterManagement.Clean_Directories_On_List_Or_All tmp_dir=/tmp
399 ClusterManagement.Start_Members_From_List_Or_All
400 BuiltIn.Wait_Until_Keyword_Succeeds 300s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} verify_restconf=True
401 ClusterManagement.Run_Bash_Command_On_List_Or_All ps -ef | grep java
403 Check_Status_Of_First_Response
404 [Arguments] ${resp_list}
405 [Documentation] Extract first item from the list, third item of the tuple and call TemplatedRequests to check the http status code.
406 # @{resp_list}[0][2] does not work
407 ${tuple} = BuiltIn.Set_Variable @{resp_list}[0]
408 TemplatedRequests.Check_Status_Code @{tuple}[2]