csit/libraries/controller/DdbCommons.robot

   1 *** Settings ***
   2 Documentation     DOMDataBroker testing: Common keywords
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This resource file implements various test cases templates.
  11 ...               FIXME: add a link to a document (when published) where the scenarios are defined
  12 Library           ${CURDIR}/../MdsalLowlevelPy.py
  13 Resource          ${CURDIR}/../ClusterAdmin.robot
  14 Resource          ${CURDIR}/../ClusterManagement.robot
  15 Resource          ${CURDIR}/../MdsalLowlevel.robot
  16 Resource          ${CURDIR}/../TemplatedRequests.robot
  17 Resource          ${CURDIR}/../ShardStability.robot
  18 Resource          ${CURDIR}/../WaitForFailure.robot
  19
  20 *** Variables ***
  21 ${SHARD_NAME}     default
  22 ${SHARD_TYPE}     config
  23 ${TRANSACTION_RATE_1K}    ${1000}
  24 ${DURATION_30S}    ${30}
  25 ${DURATION_10S}    ${10}
  26 ${ID_PREFIX}      prefix-
  27 ${TRANSACTION_TIMEOUT}    ${30}
  28 ${TRANSACTION_TIMEOUT_2X}    ${2*${TRANSACTION_TIMEOUT}}
  29 ${SIMPLE_TX}      ${False}
  30 ${CHAINED_TX}     ${True}
  31 ${HARD_TIMEOUT}    ${60}
  32 @{TRANSACTION_FAILED}    ${500}
  33 ${PREF_BASED_SHARD}    id-ints
  34 ${TEST_LOG_LEVEL}    info
  35 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller.cluster.sharding    org.opendaylight.controller.cluster.datastore
  36
  37 *** Keywords ***
  38 Explicit_Leader_Movement_Test_Templ
  39     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  40     [Documentation]    Implements explicit leader movement test scenario.
  41     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}
  42     ...    ${shard_type}
  43     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  44     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  45     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${ip_trans_as_list}    ${idx_trans_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${False}
  46     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
  47     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    30s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}
  48     ...    ${shard_type}    ${True}    ${idx_from}
  49     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  50     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  51     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  52
  53 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
  54     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
  55     [Documentation]    Implements explicit leader movement test scenario.
  56     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}!!
  57     ...    ${shard_type}
  58     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  59     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  60     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${ip_trans_as_list}    ${idx_trans_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}
  61     MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}    ${ID_PREFIX}
  62     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    30s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!
  63     ...    ${shard_type}    ${True}    ${idx_from}
  64     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  65     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  66     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  67
  68 Get_Node_Indexes_For_The_ELM_Test
  69     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}    ${shard_type}
  70     [Documentation]    Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
  71     ...    producer should be deployed.
  72     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}
  73     ${idx_from} =    BuiltIn.Set_Variable    ${leader}
  74     ${idx_to} =    BuiltIn.Set_Variable    @{follower_list}[0]
  75     ${idx_trans} =    BuiltIn.Set_Variable_If    "${leader_from}" == "remote" and "${leader_to}" == "remote"    @{follower_list}[1]    "${leader_from}" == "local"    ${leader}    "${leader_to}" == "local"
  76     ...    @{follower_list}[0]
  77     BuiltIn.Return_From_Keyword    ${idx_from}    ${idx_to}    ${idx_trans}
  78
  79 Clean_Leader_Shutdown_Test_Templ
  80     [Arguments]    ${leader_location}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  81     [Documentation]    Implements clean leader shutdown test scenario.
  82     ${removed} =    BuiltIn.Set_Variable    ${False}
  83     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}    ${shard_name}    ${shard_type}
  84     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
  85     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
  86     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${False}
  87     ClusterAdmin.Remove_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
  88     ${removed} =    BuiltIn.Set_Variable    ${True}
  89     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  90     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  91     [Teardown]    BuiltIn.Run_Keywords    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard
  92     ...    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${follower_list}
  93     ...    AND    ClusterAdmin.Add_Shard_Replica    ${actual_leader}    ${shard_name}    ${shard_type}
  94
  95 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
  96     [Arguments]    ${leader_location}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
  97     [Documentation]    Implements clean leader shutdown test scenario.
  98     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}    ${shard_name}!!    ${shard_type}
  99     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
 100     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
 101     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}
 102     ClusterAdmin.Remove_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
 103     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${follower_list}
 104     BuiltIn.Run_Keyword_And_Ignore_Error    ClusterManagement.Get_Raft_State_Of_Shard_At_Member    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index=${actual_leader}
 105     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    10s    2s    ClusterManagement.Get_Raft_State_Of_Shard_At_Member    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index=${actual_leader}
 106     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 107     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
 108     [Teardown]    BuiltIn.Run_Keywords    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard
 109     ...    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${follower_list}
 110     ...    AND    ClusterAdmin.Add_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}    ${shard_type}
 111
 112 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
 113     [Arguments]    ${leader_location}    ${shard_name}    ${shard_type}
 114     [Documentation]    Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
 115     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}
 116     ${follower_list_leangth} =    BuiltIn.Evaluate    ${NUM_ODL_SYSTEM}-1
 117     BuiltIn.Length_Should_Be    ${follower_list}    ${follower_list_leangth}
 118     ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    @{follower_list}[0]
 119     BuiltIn.Return_From_Keyword    ${producer_idx}    ${leader}    ${follower_list}
 120
 121 Leader_Isolation_Test_Templ
 122     [Arguments]    ${heal_timeout}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 123     [Documentation]    Implements leader isolation test scenario.
 124     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 125     ${producing_transactions_time} =    BuiltIn.Set_Variable    ${${heal_timeout}+60}
 126     ${all_indices} =    ClusterManagement.List_All_Indices
 127     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 128     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 129     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}    chained_flag=${SIMPLE_TX}
 130     ${date_start} =    DateTime.Get_Current_Date
 131     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 132     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 133     ${li_isolated}    BuiltIn.Set_Variable    ${True}
 134     BuiltIn.Wait_Until_Keyword_Succeeds    45s    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}    ${shard_type}    ${True}
 135     ...    ${leader}    member_index_list=${follower_list}
 136     BuiltIn.Sleep    ${heal_timeout}
 137     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 138     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 139     BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 140     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${shard_name}    ${shard_type}
 141     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 142     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${TRANSACTION_TIMEOUT}    Leader_Isolation_Heal_Within_Tt
 143     ...    ELSE    Module_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 144     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 145     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 146
 147 Leader_Isolation_PrefBasedShard_Test_Templ
 148     [Arguments]    ${heal_timeout}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 149     [Documentation]    Implements leader isolation test scenario.
 150     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 151     ${producing_transactions_time} =    BuiltIn.Set_Variable    ${${heal_timeout}+60}
 152     ${all_indices} =    ClusterManagement.List_All_Indices
 153     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}
 154     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 155     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}
 156     ${date_start} =    DateTime.Get_Current_Date
 157     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 158     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 159     ${li_isolated}    BuiltIn.Set_Variable    ${True}
 160     BuiltIn.Wait_Until_Keyword_Succeeds    45s    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!    ${shard_type}    ${True}
 161     ...    ${leader}    member_index_list=${follower_list}
 162     BuiltIn.Sleep    ${heal_timeout}
 163     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 164     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 165     BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 166     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${shard_name}!!    ${shard_type}
 167     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 168     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${TRANSACTION_TIMEOUT}    Leader_Isolation_Heal_Within_Tt
 169     ...    ELSE    Prefix_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 170     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 171     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 172
 173 Leader_Isolation_Heal_Within_Tt
 174     [Documentation]    The leader isolation test case end if the heal happens within transaction timeout. All write transaction
 175     ...    producers shoudl finish without error.
 176     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 177     : FOR    ${resp}    IN    @{resp_list}
 178     \    TemplatedRequests.Check_Status_Code    ${resp}
 179
 180 Module_Leader_Isolation_Heal_Default
 181     [Arguments]    ${isolated_node}    ${time_to_finish}
 182     [Documentation]    The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
 183     Then all write transaction producers should finish without error.
 184     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 185     BuiltIn.Log    ${resp}
 186     # TODO: check on response status code
 187     ${restart_producer_node_idx_as_list}    BuiltIn.Create_List    ${isolated_node}
 188     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 189     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
 190     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    chained_flag=${SIMPLE_TX}
 191     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 192     : FOR    ${resp}    IN    @{resp_list}
 193     \    TemplatedRequests.Check_Status_Code    ${resp}
 194
 195 Prefix_Leader_Isolation_Heal_Default
 196     [Arguments]    ${isolated_node}    ${time_to_finish}
 197     [Documentation]    The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
 198     Then all write transaction producers shoudl finish without error.
 199     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 200     BuiltIn.Log    ${resp}
 201     # TODO: check on response status code
 202     ${restart_producer_node_idx_as_list}    BuiltIn.Create_List    ${isolated_node}
 203     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 204     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
 205     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX}    ${time_to_finish}    ${TRANSACTION_RATE_1K}
 206     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 207     : FOR    ${resp}    IN    @{resp_list}
 208     \    TemplatedRequests.Check_Status_Code    ${resp}
 209
 210 Client_Isolation_Test_Templ
 211     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 212     [Documentation]    Implements client isolation test scenario.
 213     ${all_indices} =    ClusterManagement.List_All_Indices
 214     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 215     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 216     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 217     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 218     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 219     ${client_node_idx_as_list}    BuiltIn.Create_List    ${client_node_dst}
 220     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${client_node_ip_as_list}    ${client_node_idx_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${trans_chain_flag}
 221     ${start_date}    DateTime.Get_Current_Date
 222     ${timeout_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${TRANSACTION_TIMEOUT}
 223     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${HARD_TIMEOUT}
 224     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 225     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${TRANSACTION_TIMEOUT}    1s    Ongoing_Transactions_Not_Failed_Yet
 226     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    3s    1s    Ongoing_Transactions_Failed
 227     ${abort_time}    Get_Seconds_To_Time    ${abort_date}
 228     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${abort_time}    1s    Verify_Client_Aborted    ${False}
 229     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    3s    1s    Verify_Client_Aborted    ${True}
 230     [Teardown]    BuiltIn.Run Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 231     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 232     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 233
 234 Client_Isolation_PrefBasedShard_Test_Templ
 235     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 236     [Documentation]    Implements client isolation test scenario.
 237     ${all_indices} =    ClusterManagement.List_All_Indices
 238     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 239     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 240     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 241     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 242     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 243     ${client_node_idx_as_list}    BuiltIn.Create_List    ${client_node_dst}
 244     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${client_node_ip_as_list}    ${client_node_idx_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}
 245     ${start_date}    DateTime.Get_Current_Date
 246     ${timeout_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${TRANSACTION_TIMEOUT}
 247     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${HARD_TIMEOUT}
 248     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 249     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${TRANSACTION_TIMEOUT}    1s    Ongoing_Transactions_Not_Failed_Yet
 250     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    3s    1s    Ongoing_Transactions_Failed
 251     ${abort_time}    Get_Seconds_To_Time    ${abort_date}
 252     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${abort_time}    1s    Verify_Client_Aborted    ${False}
 253     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    3s    1s    Verify_Client_Aborted    ${True}
 254     [Teardown]    BuiltIn.Run Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 255     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 256     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 257
 258 Ongoing_Transactions_Not_Failed_Yet
 259     [Documentation]    Verify that no write-transaction rpc finished, means they are still running.
 260     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 261     BuiltIn.Should_Be_Equal    ${None}    ${resp}    ${resp} not expected.
 262
 263 Ongoing_Transactions_Failed
 264     [Documentation]    Verify if write-transaction failed.
 265     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 266     Check_Status_Code    ${resp}    explicit_status_codes=${TRANSACTION_FAILED}
 267
 268 Get_Seconds_To_Time
 269     [Arguments]    ${date_in_future}
 270     [Documentation]    Return number of seconds remaining to ${date_in_future}.
 271     ${date_now} =    DateTime.Get_Current_Date
 272     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
 273     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 274
 275 Remote_Listener_Test_Templ
 276     [Arguments]    ${listener_node_role}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 277     [Documentation]    Implements remote listener test scenario.
 278     ${all_indices} =    ClusterManagement.List_All_Indices
 279     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 280     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 281     ${follower2} =    Collections.Get_From_List    ${follower_list}    ${1}
 282     ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 283     MdsalLowlevel.Subscribe_Dtcl    ${listener_node_dst}
 284     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 285     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${DURATION_10S}    ${TRANSACTION_RATE_1K}    chained_flag=${SIMPLE_TX}
 286     ClusterAdmin.Make_Leader_Local    ${follower1}    ${shard_name}    ${shard_type}
 287     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 288     : FOR    ${resp}    IN    @{resp_list}
 289     \    TemplatedRequests.Check_Status_Code    ${resp}
 290     [Teardown]    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}
 291
 292 Remote_Listener_PrefBasedShard_Test_Templ
 293     [Arguments]    ${listener_node_role}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 294     [Documentation]    Implements listener isolation test scenario.
 295     ${all_indices} =    ClusterManagement.List_All_Indices
 296     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 297     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 298     ${follower2} =    Collections.Get_From_List    ${follower_list}    ${1}
 299     ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 300     MdsalLowlevel.Subscribe_Dtcl    ${listener_node_dst}
 301     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 302     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${DURATION_10S}    ${TRANSACTION_RATE_1K}
 303     ClusterAdmin.Make_Leader_Local    ${follower1}    ${shard_name}    ${shard_type}
 304     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 305     : FOR    ${resp}    IN    @{resp_list}
 306     \    TemplatedRequests.Check_Status_Code    ${resp}
 307     [Teardown]    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}
 308
 309 Create_Prefix_Based_Shard_And_Verify
 310     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 311     [Documentation]    Create prefix based shard with replicas on all nodes
 312     ${all_indices} =    ClusterManagement.List_All_Indices
 313     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 314     MdsalLowlevel.Create_Prefix_Shard    ${node_to_trigger}    ${prefix}    ${all_indices}
 315     BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${prefix}!!    shard_type=${SHARD_TYPE}    member_index_list=${all_indices}
 316
 317 Remove_Prefix_Based_Shard_And_Verify
 318     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 319     [Documentation]    Remove prefix based shard with all replicas
 320     ${all_indices} =    ClusterManagement.List_All_Indices
 321     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 322     MdsalLowlevel.Remove_Prefix_Shard    ${node_to_trigger}    ${prefix}
 323     : FOR    ${idx}    IN    @{all_indices}
 324     \    BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    Verify_Shard_Replica_Removed    ${idx}    ${prefix}!!
 325     \    ...    ${SHARD_TYPE}
 326
 327 Verify_Shard_Replica_Removed
 328     [Arguments]    ${member_index}    ${shard_name}    ${shard_type}
 329     [Documentation]    Verify that shard is removed. Jolokia return 404 for shard memeber.
 330     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 331     ${type_class} =    Resolve_Shard_Type_Class    shard_type=${shard_type}
 332     ${uri} =    BuiltIn.Set_Variable    /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
 333     ${text}    TemplatedRequests.Get_From_Uri    uri=${uri}    session=${session}
 334     BuiltIn.Should_Contain    ${text}    "status":404    javax.management.InstanceNotFoundException