: FOR ${shard_details} IN @{shard_list}
\ ${shard_name} ${shard_type} String.Split_String ${shard_details} separator=:
\ ${leader} ${followers} ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${member_index_list}
+ \ ... verify_restconf=False
\ Collections.Sort_List ${followers}
\ Collections.Set_To_Dictionary ${shards_details} ${shard_name}_${shard_type}_leader=${leader}
\ Collections.Set_To_Dictionary ${shards_details} ${shard_name}_${shard_type}_followers=${followers}
Verify_Shard_Leader_Located_As_Expected
[Arguments] ${expected_leader_idx}
[Documentation] Verify default/topology config/operational shard leader location is as expected
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=default shard_type=config
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=default shard_type=config verify_restconf=False
BuiltIn.Should_Be_Equal_As_Numbers ${expected_leader_idx} ${leader}
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=default shard_type=operational
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=default shard_type=operational verify_restconf=False
BuiltIn.Should_Be_Equal_As_Numbers ${expected_leader_idx} ${leader}
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=topology shard_type=config
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=topology shard_type=config verify_restconf=False
BuiltIn.Should_Be_Equal_As_Numbers ${expected_leader_idx} ${leader}
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=topology shard_type=operational
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=topology shard_type=operational verify_restconf=False
BuiltIn.Should_Be_Equal_As_Numbers ${expected_leader_idx} ${leader}
[Arguments] ${leader_from} ${leader_to} ${shard_name} ${shard_type}
[Documentation] Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
... producer should be deployed.
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} verify_restconf=False
${idx_from} = BuiltIn.Set_Variable ${leader}
${idx_to} = BuiltIn.Set_Variable @{follower_list}[0]
${idx_trans} = BuiltIn.Set_Variable_If "${leader_from}" == "remote" and "${leader_to}" == "remote" @{follower_list}[1] "${leader_from}" == "local" ${leader} "${leader_to}" == "local"
${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
TemplatedRequests.Check_Status_Code @{resp_list}[0]
[Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard
- ... shard_name=${shard_name} shard_type=${shard_type} member_index_list=${follower_list}
+ ... shard_name=${shard_name} shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
... AND ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
- ... shard_type=${shard_type}
+ ... verify_restconf=False shard_type=${shard_type}
Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
[Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K}
ClusterAdmin.Remove_Prefix_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
+ ... verify_restconf=False
BuiltIn.Run_Keyword_And_Ignore_Error ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 10s 2s ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader}
${resp_list} = MdsalLowlevelPy.Wait_For_Transactions
TemplatedRequests.Check_Status_Code @{resp_list}[0]
[Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard
- ... shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list}
+ ... shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list} verify_restconf=False
... AND ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type}
... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
- ... shard_type=${shard_type}
+ ... verify_restconf=False shard_type=${shard_type}
Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
[Arguments] ${leader_location} ${shard_name} ${shard_type}
[Documentation] Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} verify_restconf=False
${follower_list_leangth} = BuiltIn.Evaluate ${NUM_ODL_SYSTEM}-1
BuiltIn.Length_Should_Be ${follower_list} ${follower_list_leangth}
${producer_idx} = BuiltIn.Set_Variable_If "${leader_location}" == "local" ${leader} @{follower_list}[0]
${li_isolated} BuiltIn.Set_Variable ${False}
${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX}
${date_start} = DateTime.Get_Current_Date
ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
${li_isolated} BuiltIn.Set_Variable ${False}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
- BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type} verify_restconf=False
${time_to_finish} = Get_Seconds_To_Time ${date_end}
BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
${li_isolated} BuiltIn.Set_Variable ${False}
${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices}
MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K}
${date_start} = DateTime.Get_Current_Date
ClusterManagement.Rejoin_Member_From_List_Or_All ${leader}
${li_isolated} BuiltIn.Set_Variable ${False}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST}
- BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type}
+ BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type} verify_restconf=False
${time_to_finish} = Get_Seconds_To_Time ${date_end}
BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt
... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish}
[Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE}
[Documentation] Implements client isolation test scenario.
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
[Arguments] ${listener_node_role} ${isolated_transactions_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE}
[Documentation] Implements client isolation test scenario.
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst}
[Documentation] Implements remote listener test scenario.
${subscribed} = BuiltIn.Set_Variable ${False}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
${follower2} = Collections.Get_From_List ${follower_list} ${1}
${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
[Documentation] Implements listener isolation test scenario.
${subscribed} = BuiltIn.Set_Variable ${False}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
${follower2} = Collections.Get_From_List ${follower_list} ${1}
${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1}
${node_to_trigger} = Collections.Get_From_List ${all_indices} ${0}
MdsalLowlevel.Create_Prefix_Shard ${node_to_trigger} ${prefix} ${all_indices}
BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${prefix}!! shard_type=${SHARD_TYPE} member_index_list=${all_indices}
+ ... verify_restconf=False
Remove_Prefix_Based_Shard_And_Verify
[Arguments] ${prefix}=${PREF_BASED_SHARD}
${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME}
${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
${all_indices} = ClusterManagement.List_All_Indices
- ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} member_index_list=${all_indices}
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type}
${leader} ${follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}
${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME}
${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
${all_indices} = ClusterManagement.List_All_Indices
- ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
ClusterAdmin.Remove_Shard_Replica ${old_leader} ${shard_name} member-${old_leader} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${old_leader} ${shard_name} ${shard_type}
${actual_leader} ${actual_follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}
- ... shard_type=${shard_type} member_index_list=${follower_list}
+ ... verify_restconf=False shard_type=${shard_type} member_index_list=${follower_list}
BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_leader} ${actual_leader}
ClusterAdmin.Add_Shard_Replica ${old_leader} ${shard_name} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ... verify_restconf=False
Remove_Follower_Shard_Replica_And_Add_It_Back
[Documentation] Remove and add shard replica adn verify it.
${shard_name} = BuiltIn.Set_Variable ${SHARD_NAME}
${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
ClusterAdmin.Remove_Shard_Replica ${follower1} ${shard_name} member-${follower1} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${follower1} ${shard_name} ${shard_type}
ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${False} ${leader} member_index_list=${new_indices_list}
ClusterAdmin.Add_Shard_Replica ${follower1} ${shard_name} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices}
+ ... verify_restconf=False
Write_Transactions
[Documentation] Write transactions.
${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD}
${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
${all_indices} = ClusterManagement.List_All_Indices
- ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
MdsalLowlevel.Become_Prefix_Leader ${follower1} ${shard_name} ${ID_PREFIX}
${leader} ${follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!!
${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD}
${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
${all_indices} = ClusterManagement.List_All_Indices
- ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${old_leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
ClusterAdmin.Remove_Prefix_Shard_Replica ${old_leader} ${shard_name} member-${old_leader} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s Verify_Shard_Replica_Removed ${old_leader} ${shard_name}!! ${shard_type}
${actual_leader} ${actual_follower_list} = BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!!
- ... shard_type=${shard_type} member_index_list=${follower_list}
+ ... verify_restconf=False shard_type=${shard_type} member_index_list=${follower_list}
ClusterAdmin.Add_Prefix_Shard_Replica ${old_leader} ${shard_name} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ... verify_restconf=False
Remove_Follower_Prefix_Shard_Replica_And_Add_It_Back
[Documentation] Remove and add shard replica adn verify it.
${shard_name} = BuiltIn.Set_Variable ${PREF_BASED_SHARD}
${shard_type} = BuiltIn.Set_Variable ${SHARD_TYPE}
${all_indices} = ClusterManagement.List_All_Indices
- ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} verify_restconf=False
${follower1} = Collections.Get_From_List ${follower_list} ${0}
ClusterAdmin.Remove_Prefix_Shard_Replica ${follower1} ${shard_name} member-${follower1} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s DdbCommons.Verify_Shard_Replica_Removed ${follower1} ${shard_name}!! ${shard_type}
ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${False} ${leader} member_index_list=${new_indices_list}
ClusterAdmin.Add_Prefix_Shard_Replica ${follower1} ${shard_name} ${shard_type}
BuiltIn.Wait_Until_Keyword_Succeeds 60s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices}
+ ... verify_restconf=False