X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2Fcontroller%2FDdbCommons.robot;h=c203504a9eb6cfc99be2205f2c0ad1815b7d0ce1;hb=e1fd286cfcd8de767535d6fc126734e9327e27ce;hp=5c013af78d1dbf18f84b1f81604f491fc5f0f972;hpb=86d8313fcc990a1f8e8b335e1676fcc032cd6e89;p=integration%2Ftest.git diff --git a/csit/libraries/controller/DdbCommons.robot b/csit/libraries/controller/DdbCommons.robot index 5c013af78d..c203504a9e 100644 --- a/csit/libraries/controller/DdbCommons.robot +++ b/csit/libraries/controller/DdbCommons.robot @@ -14,12 +14,14 @@ Resource ${CURDIR}/../ClusterAdmin.robot Resource ${CURDIR}/../ClusterManagement.robot Resource ${CURDIR}/../MdsalLowlevel.robot Resource ${CURDIR}/../TemplatedRequests.robot +Resource ${CURDIR}/../ShardStability.robot Resource ${CURDIR}/../WaitForFailure.robot *** Variables *** ${SHARD_NAME} default ${SHARD_TYPE} config ${TRANSACTION_RATE_1K} ${1000} +${DURATION_90S} ${90} #TODO: Rename to hint at the goal of waiting, instead of its default value. ${DURATION_30S} ${30} ${DURATION_10S} ${10} ${ID_PREFIX} prefix- @@ -27,7 +29,9 @@ ${TRANSACTION_TIMEOUT} ${30} ${TRANSACTION_TIMEOUT_2X} ${2*${TRANSACTION_TIMEOUT}} ${SIMPLE_TX} ${False} ${CHAINED_TX} ${True} -${HARD_TIMEOUT} ${60} +${ISOLATED_TRANS_TRUE} ${True} +${ISOLATED_TRANS_FALSE} ${False} +${HARD_TIMEOUT} ${2*${TRANSACTION_TIMEOUT}} @{TRANSACTION_FAILED} ${500} ${PREF_BASED_SHARD} id-ints ${TEST_LOG_LEVEL} info @@ -41,9 +45,9 @@ Explicit_Leader_Movement_Test_Templ ... ${shard_type} ${ip_trans_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${idx_trans}_IP} ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX} ClusterAdmin.Make_Leader_Local ${idx_to} ${shard_name} ${shard_type} - ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 10s 1s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} + ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ... ${shard_type} ${True} ${idx_from} BuiltIn.Should_Be_Equal ${idx_to} ${new_leader} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions @@ -58,7 +62,7 @@ Explicit_Leader_Movement_PrefBasedShard_Test_Templ ${idx_trans_as_list} = BuiltIn.Create_List ${idx_trans} MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${ip_trans_as_list} ${idx_trans_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} MdsalLowlevel.Become_Prefix_Leader ${idx_to} ${shard_name} ${ID_PREFIX} - ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 10s 1s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! + ${new_leader} ${new_followers} = BuiltIn.Wait_Until_Keyword_Succeeds 30s 5s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ... ${shard_type} ${True} ${idx_from} BuiltIn.Should_Be_Equal ${idx_to} ${new_leader} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions @@ -82,12 +86,16 @@ Clean_Leader_Shutdown_Test_Templ ${producer_idx} ${actual_leader} ${follower_list} = Get_Node_Indexes_For_Clean_Leader_Shutdown_Test ${leader_location} ${shard_name} ${shard_type} ${producer_ip_as_list} = BuiltIn.Create_List ${ODL_SYSTEM_${producer_idx}_IP} ${producer_idx_as_list} = BuiltIn.Create_List ${producer_idx} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${False} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${producer_ip_as_list} ${producer_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX} ClusterAdmin.Remove_Shard_Replica ${actual_leader} ${shard_name} member-${actual_leader} ${shard_type} ${removed} = BuiltIn.Set_Variable ${True} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions TemplatedRequests.Check_Status_Code @{resp_list}[0] - [Teardown] ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type} + [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard + ... shard_name=${shard_name} shard_type=${shard_type} member_index_list=${follower_list} + ... AND ClusterAdmin.Add_Shard_Replica ${actual_leader} ${shard_name} ${shard_type} + ... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} + ... shard_type=${shard_type} Clean_Leader_Shutdown_PrefBasedShard_Test_Templ [Arguments] ${leader_location} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} @@ -102,7 +110,11 @@ Clean_Leader_Shutdown_PrefBasedShard_Test_Templ WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 10s 2s ClusterManagement.Get_Raft_State_Of_Shard_At_Member shard_name=${shard_name}!! shard_type=${shard_type} member_index=${actual_leader} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions TemplatedRequests.Check_Status_Code @{resp_list}[0] - [Teardown] ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type} + [Teardown] BuiltIn.Run_Keywords BuiltIn.Run_Keyword_And_Ignore_Error BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard + ... shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${follower_list} + ... AND ClusterAdmin.Add_Prefix_Shard_Replica ${actual_leader} ${shard_name} ${shard_type} + ... AND BuiltIn.Wait_Until_Keyword_Succeeds 30s 3s ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! + ... shard_type=${shard_type} Get_Node_Indexes_For_Clean_Leader_Shutdown_Test [Arguments] ${leader_location} ${shard_name} ${shard_type} @@ -116,26 +128,33 @@ Get_Node_Indexes_For_Clean_Leader_Shutdown_Test Leader_Isolation_Test_Templ [Arguments] ${heal_timeout} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE} [Documentation] Implements leader isolation test scenario. + ${li_isolated} BuiltIn.Set_Variable ${False} ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60} ${all_indices} = ClusterManagement.List_All_Indices ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${producing_transactions_time} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX} ${date_start} = DateTime.Get_Current_Date ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time} ClusterManagement.Isolate_Member_From_List_Or_All ${leader} - BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True} + ${li_isolated} BuiltIn.Set_Variable ${True} + BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True} ... ${leader} member_index_list=${follower_list} BuiltIn.Sleep ${heal_timeout} ClusterManagement.Rejoin_Member_From_List_Or_All ${leader} - BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type} + ${li_isolated} BuiltIn.Set_Variable ${False} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} + BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name} ${shard_type} ${time_to_finish} = Get_Seconds_To_Time ${date_end} BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt ... ELSE Module_Leader_Isolation_Heal_Default ${leader} ${time_to_finish} + [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader} + ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} Leader_Isolation_PrefBasedShard_Test_Templ [Arguments] ${heal_timeout} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} [Documentation] Implements leader isolation test scenario. + ${li_isolated} BuiltIn.Set_Variable ${False} ${producing_transactions_time} = BuiltIn.Set_Variable ${${heal_timeout}+60} ${all_indices} = ClusterManagement.List_All_Indices ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} @@ -144,14 +163,19 @@ Leader_Isolation_PrefBasedShard_Test_Templ ${date_start} = DateTime.Get_Current_Date ${date_end} = DateTime.Add_Time_To_Date ${date_start} ${producing_transactions_time} ClusterManagement.Isolate_Member_From_List_Or_All ${leader} - BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True} + ${li_isolated} BuiltIn.Set_Variable ${True} + BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True} ... ${leader} member_index_list=${follower_list} BuiltIn.Sleep ${heal_timeout} ClusterManagement.Rejoin_Member_From_List_Or_All ${leader} - BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type} + ${li_isolated} BuiltIn.Set_Variable ${False} + BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} + BuiltIn.Wait_Until_Keyword_Succeeds 15s 2s ClusterManagement.Get_Leader_And_Followers_For_Shard ${shard_name}!! ${shard_type} ${time_to_finish} = Get_Seconds_To_Time ${date_end} BuiltIn.Run_Keyword_If ${heal_timeout} < ${TRANSACTION_TIMEOUT} Leader_Isolation_Heal_Within_Tt ... ELSE Prefix_Leader_Isolation_Heal_Default ${leader} ${time_to_finish} + [Teardown] BuiltIn.Run_Keyword_If ${li_isolated} BuiltIn.Run_Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${leader} + ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} Leader_Isolation_Heal_Within_Tt [Documentation] The leader isolation test case end if the heal happens within transaction timeout. All write transaction @@ -163,14 +187,15 @@ Leader_Isolation_Heal_Within_Tt Module_Leader_Isolation_Heal_Default [Arguments] ${isolated_node} ${time_to_finish} [Documentation] The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted. - Then all write transaction producers should finish without error. + ... Then all write transaction producers should finish without error. ${resp} = MdsalLowlevelPy.Get_Next_Transactions_Response BuiltIn.Log ${resp} # TODO: check on response status code ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node} ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node} ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX} + ... reset_globals=${False} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} @@ -185,7 +210,7 @@ Prefix_Leader_Isolation_Heal_Default ${restart_producer_node_idx_as_list} BuiltIn.Create_List ${isolated_node} ${restart_producer_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${isolated_node} ${restart_producer_node_ip_as_list} BuiltIn.Create_List ${restart_producer_node_ip} - MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${restart_producer_node_ip_as_list} ${restart_producer_node_idx_as_list} ${ID_PREFIX} ${time_to_finish} ${TRANSACTION_RATE_1K} reset_globals=${False} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} @@ -200,40 +225,42 @@ Client_Isolation_Test_Templ ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst} ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip} ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_90S} ${TRANSACTION_RATE_1K} chained_flag=${trans_chain_flag} ${start_date} DateTime.Get_Current_Date ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT} ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT} ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst} - WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet - WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed - ${abort_time} Get_Seconds_To_Time ${abort_date} - WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False} - WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True} + ${trans_timeout} = Get_Seconds_To_Time ${timeout_date} + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${trans_timeout} 1s Ongoing_Transactions_Not_Failed_Yet + ${hard_timeout} = Get_Seconds_To_Time ${abort_date} + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${hard_timeout} 1s Ongoing_Transactions_Not_Failed_Yet + BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst} + ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} ... AND MdsalLowlevelPy.Wait_For_Transactions Client_Isolation_PrefBasedShard_Test_Templ - [Arguments] ${listener_node_role} ${trans_chain_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} + [Arguments] ${listener_node_role} ${isolated_transactions_flag} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} [Documentation] Implements client isolation test scenario. ${all_indices} = ClusterManagement.List_All_Indices - ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} ${follower1} = Collections.Get_From_List ${follower_list} ${0} ${client_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} ${client_node_ip} = ClusterManagement.Resolve_IP_Address_For_Member ${client_node_dst} ${client_node_ip_as_list} BuiltIn.Create_List ${client_node_ip} ${client_node_idx_as_list} BuiltIn.Create_List ${client_node_dst} - MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_30S} ${TRANSACTION_RATE_1K} + MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${client_node_ip_as_list} ${client_node_idx_as_list} ${ID_PREFIX} ${DURATION_90S} ${TRANSACTION_RATE_1K} isolated_transactions_flag=${isolated_transactions_flag} ${start_date} DateTime.Get_Current_Date ${timeout_date} = DateTime.Add_Time_To_Date ${start_date} ${TRANSACTION_TIMEOUT} ${abort_date} = DateTime.Add_Time_To_Date ${start_date} ${HARD_TIMEOUT} ClusterManagement.Isolate_Member_From_List_Or_All ${client_node_dst} - WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${TRANSACTION_TIMEOUT} 1s Ongoing_Transactions_Not_Failed_Yet - WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Ongoing_Transactions_Failed - ${abort_time} Get_Seconds_To_Time ${abort_date} - WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${abort_time} 1s Verify_Client_Aborted ${False} - WaitForFailure.Confirm_Keyword_Fails_Within_Timeout 3s 1s Verify_Client_Aborted ${True} + ${trans_timeout} = Get_Seconds_To_Time ${timeout_date} + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${trans_timeout} 1s Ongoing_Transactions_Not_Failed_Yet + ${hard_timeout} = Get_Seconds_To_Time ${abort_date} + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${hard_timeout} 1s Ongoing_Transactions_Not_Failed_Yet + BuiltIn.Wait_Until_Keyword_Succeeds 20s 2s Ongoing_Transactions_Failed [Teardown] BuiltIn.Run Keywords ClusterManagement.Rejoin_Member_From_List_Or_All ${client_node_dst} + ... AND BuiltIn.Wait_Until_Keyword_Succeeds 60s 10s ShardStability.Shards_Stability_Get_Details ${DEFAULT_SHARD_LIST} ... AND MdsalLowlevelPy.Wait_For_Transactions Ongoing_Transactions_Not_Failed_Yet @@ -256,36 +283,50 @@ Get_Seconds_To_Time Remote_Listener_Test_Templ [Arguments] ${listener_node_role} ${shard_name}=${SHARD_NAME} ${shard_type}=${SHARD_TYPE} [Documentation] Implements remote listener test scenario. + ${subscribed} = BuiltIn.Set_Variable ${False} ${all_indices} = ClusterManagement.List_All_Indices ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} ${follower1} = Collections.Get_From_List ${follower_list} ${0} ${follower2} = Collections.Get_From_List ${follower_list} ${1} ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst} + ${subscribed} = BuiltIn.Set_Variable ${True} ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} - MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${SIMPLE_TX} + MdsalLowlevelPy.Start_Write_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} chained_flag=${CHAINED_TX} ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type} + BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name} ${shard_type} ${True} + ... ${leader} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} - [Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst} + ${copy_matches} = MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst} + ${subscribed} = BuiltIn.Set_Variable ${False} + BuiltIn.Should_Be_True ${copy_matches} + [Teardown] BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst} Remote_Listener_PrefBasedShard_Test_Templ [Arguments] ${listener_node_role} ${shard_name}=${PREF_BASED_SHARD} ${shard_type}=${SHARD_TYPE} [Documentation] Implements listener isolation test scenario. + ${subscribed} = BuiltIn.Set_Variable ${False} ${all_indices} = ClusterManagement.List_All_Indices - ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${all_indices} + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name}!! shard_type=${shard_type} member_index_list=${all_indices} ${follower1} = Collections.Get_From_List ${follower_list} ${0} ${follower2} = Collections.Get_From_List ${follower_list} ${1} ${listener_node_dst} = BuiltIn.Set_Variable_If "${listener_node_role}" == "leader" ${leader} ${follower1} - MdsalLowlevel.Subscribe_Dtcl ${listener_node_dst} + MdsalLowlevel.Subscribe_Ddtl ${listener_node_dst} + ${subscribed} = BuiltIn.Set_Variable ${True} ${all_ip_list} = ClusterManagement.Resolve_IP_Address_For_Members ${all_indices} MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes ${all_ip_list} ${all_indices} ${ID_PREFIX} ${DURATION_10S} ${TRANSACTION_RATE_1K} - ClusterAdmin.Make_Leader_Local ${follower1} ${shard_name} ${shard_type} + MdsalLowlevel.Become_Prefix_Leader ${follower1} ${shard_name} ${ID_PREFIX} + BuiltIn.Wait_Until_Keyword_Succeeds 45s 2s ClusterManagement.Verify_Shard_Leader_Elected ${shard_name}!! ${shard_type} ${True} + ... ${leader} ${resp_list} = MdsalLowlevelPy.Wait_For_Transactions : FOR ${resp} IN @{resp_list} \ TemplatedRequests.Check_Status_Code ${resp} - [Teardown] MdsalLowlevel.Unsubscribe_Dtcl ${listener_node_dst} + ${copy_matches} = MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst} + ${subscribed} = BuiltIn.Set_Variable ${False} + BuiltIn.Should_Be_True ${copy_matches} + [Teardown] BuiltIn.Run_Keyword_If ${subscribed} MdsalLowlevel.Unsubscribe_Ddtl ${listener_node_dst} Create_Prefix_Based_Shard_And_Verify [Arguments] ${prefix}=${PREF_BASED_SHARD}