csit/libraries/controller/DdbCommons.robot

   1 *** Settings ***
   2 Documentation     DOMDataBroker testing: Common keywords
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This resource file implements various test cases templates.
  11 ...               FIXME: add a link to a document (when published) where the scenarios are defined
  12 Library           ${CURDIR}/../MdsalLowlevelPy.py
  13 Resource          ${CURDIR}/../ClusterAdmin.robot
  14 Resource          ${CURDIR}/../ClusterManagement.robot
  15 Resource          ${CURDIR}/../KarafKeywords.robot
  16 Resource          ${CURDIR}/../MdsalLowlevel.robot
  17 Resource          ${CURDIR}/../TemplatedRequests.robot
  18 Resource          ${CURDIR}/../ShardStability.robot
  19 Resource          ${CURDIR}/../WaitForFailure.robot
  20
  21 *** Variables ***
  22 ${SHARD_NAME}     default
  23 ${SHARD_TYPE}     config
  24 ${TRANSACTION_RATE_1K}    ${1000}
  25 ${DURATION_90S}    ${90}    #TODO: Rename to hint at the goal of waiting, instead of its default value.
  26 ${DURATION_30S}    ${30}
  27 ${DURATION_10S}    ${10}
  28 ${ID_PREFIX}      prefix-
  29 ${TRANSACTION_TIMEOUT}    ${30}
  30 ${TRANSACTION_TIMEOUT_2X}    ${2*${TRANSACTION_TIMEOUT}}
  31 ${SIMPLE_TX}      ${False}
  32 ${CHAINED_TX}     ${True}
  33 ${ISOLATED_TRANS_TRUE}    ${True}
  34 ${ISOLATED_TRANS_FALSE}    ${False}
  35 ${HARD_TIMEOUT}    ${120}
  36 @{TRANSACTION_FAILED}    ${500}
  37 ${PREF_BASED_SHARD}    id-ints
  38 ${TEST_LOG_LEVEL}    info
  39 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller.cluster.sharding    org.opendaylight.controller.cluster.datastore
  40 ${HEAL_WITHIN_TRANS_TIMEOUT}    ${0}
  41
  42 *** Keywords ***
  43 Explicit_Leader_Movement_Test_Templ
  44     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  45     [Documentation]    Implements explicit leader movement test scenario.
  46     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}
  47     ...    ${shard_type}
  48     KarafKeywords.Log_Message_To_Controller_Karaf    Starting leader movement from node${idx_from} to node${idx_to}, transaction producer at node${idx_trans}.
  49     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  50     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  51     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${ip_trans_as_list}    ${idx_trans_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
  52     BuiltIn.Sleep    5s
  53     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
  54     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    30s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}
  55     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
  56     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  57     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  58     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  59
  60 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
  61     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
  62     [Documentation]    Implements explicit leader movement test scenario.
  63     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}!!
  64     ...    ${shard_type}
  65     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  66     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  67     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${ip_trans_as_list}    ${idx_trans_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}
  68     BuiltIn.Sleep    5s
  69     MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}    ${ID_PREFIX}
  70     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    30s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!
  71     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
  72     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  73     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  74     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  75
  76 Get_Node_Indexes_For_The_ELM_Test
  77     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}    ${shard_type}
  78     [Documentation]    Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
  79     ...    producer should be deployed.
  80     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    verify_restconf=False
  81     ${idx_from} =    BuiltIn.Set_Variable    ${leader}
  82     ${idx_to} =    BuiltIn.Set_Variable    @{follower_list}[0]
  83     ${idx_trans} =    BuiltIn.Set_Variable_If    "${leader_from}" == "remote" and "${leader_to}" == "remote"    @{follower_list}[1]    "${leader_from}" == "local"    ${leader}    "${leader_to}" == "local"
  84     ...    @{follower_list}[0]
  85     BuiltIn.Return_From_Keyword    ${idx_from}    ${idx_to}    ${idx_trans}
  86
  87 Clean_Leader_Shutdown_Test_Templ
  88     [Arguments]    ${leader_location}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  89     [Documentation]    Implements clean leader shutdown test scenario.
  90     ${removed} =    BuiltIn.Set_Variable    ${False}
  91     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}    ${shard_name}    ${shard_type}
  92     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
  93     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
  94     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
  95     BuiltIn.Sleep    5s
  96     ClusterAdmin.Remove_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
  97     ${removed} =    BuiltIn.Set_Variable    ${True}
  98     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  99     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
 100     [Teardown]    BuiltIn.Run_Keywords    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard
 101     ...    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
 102     ...    AND    ClusterAdmin.Add_Shard_Replica    ${actual_leader}    ${shard_name}    ${shard_type}
 103     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}
 104     ...    verify_restconf=False    shard_type=${shard_type}
 105
 106 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
 107     [Arguments]    ${leader_location}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 108     [Documentation]    Implements clean leader shutdown test scenario.
 109     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}    ${shard_name}!!    ${shard_type}
 110     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
 111     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
 112     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}
 113     BuiltIn.Sleep    5s
 114     ClusterAdmin.Remove_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
 115     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${follower_list}
 116     ...    verify_restconf=False
 117     BuiltIn.Run_Keyword_And_Ignore_Error    ClusterManagement.Get_Raft_State_Of_Shard_At_Member    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index=${actual_leader}
 118     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    10s    2s    ClusterManagement.Get_Raft_State_Of_Shard_At_Member    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index=${actual_leader}
 119     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 120     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
 121     [Teardown]    BuiltIn.Run_Keywords    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard
 122     ...    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
 123     ...    AND    ClusterAdmin.Add_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}    ${shard_type}
 124     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!
 125     ...    verify_restconf=False    shard_type=${shard_type}
 126
 127 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
 128     [Arguments]    ${leader_location}    ${shard_name}    ${shard_type}
 129     [Documentation]    Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
 130     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    verify_restconf=False
 131     ${follower_list_leangth} =    BuiltIn.Evaluate    ${NUM_ODL_SYSTEM}-1
 132     BuiltIn.Length_Should_Be    ${follower_list}    ${follower_list_leangth}
 133     ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    @{follower_list}[0]
 134     BuiltIn.Return_From_Keyword    ${producer_idx}    ${leader}    ${follower_list}
 135
 136 Leader_Isolation_Test_Templ
 137     [Arguments]    ${heal_timeout}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 138     [Documentation]    Implements leader isolation test scenario.
 139     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 140     ${producing_transactions_time} =    BuiltIn.Set_Variable    ${${heal_timeout}+60}
 141     ${all_indices} =    ClusterManagement.List_All_Indices
 142     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 143     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 144     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
 145     BuiltIn.Sleep    5s
 146     ${date_start} =    DateTime.Get_Current_Date
 147     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 148     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${leader}
 149     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 150     ${li_isolated}    BuiltIn.Set_Variable    ${True}
 151     BuiltIn.Wait_Until_Keyword_Succeeds    45s    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}    ${shard_type}    ${True}
 152     ...    ${leader}    member_index_list=${follower_list}
 153     ${date_leader_elected} =    DateTime.Get_Current_Date
 154     ${delta} =    DateTime.Subtract_Date_From_Date    ${date_leader_elected}    ${date_start}
 155     # TODO: Consider extracting the block which depends on ${heal_timeout} value into two keywords, and have just one If to decide which one to call.
 156     BuiltIn.Run_Keyword_If    ${heal_timeout}==${HEAL_WITHIN_TRANS_TIMEOUT} and ${delta} > ${TRANSACTION_TIMEOUT}    BuiltIn.Fail    We are not able to heal the cluster within transaction timout because new leader election took more time.
 157     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 158     BuiltIn.Should_Be_Equal    ${resp}    ${NONE}    No response expected, received ${resp}
 159     BuiltIn.Sleep    ${heal_timeout}
 160     KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${leader}
 161     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 162     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 163     BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 164     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${shard_name}    ${shard_type}    verify_restconf=False
 165     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 166     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${TRANSACTION_TIMEOUT}    Leader_Isolation_Heal_Within_Tt
 167     ...    ELSE    Module_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 168     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 169     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 170
 171 Leader_Isolation_PrefBasedShard_Test_Templ
 172     [Arguments]    ${heal_timeout}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 173     [Documentation]    Implements leader isolation test scenario.
 174     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 175     ${producing_transactions_time} =    BuiltIn.Set_Variable    ${${heal_timeout}+60}
 176     ${all_indices} =    ClusterManagement.List_All_Indices
 177     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 178     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 179     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}
 180     BuiltIn.Sleep    5s
 181     ${date_start} =    DateTime.Get_Current_Date
 182     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 183     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${leader}
 184     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 185     ${li_isolated}    BuiltIn.Set_Variable    ${True}
 186     BuiltIn.Wait_Until_Keyword_Succeeds    45s    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!    ${shard_type}    ${True}
 187     ...    ${leader}    member_index_list=${follower_list}
 188     ${date_leader_elected} =    DateTime.Get_Current_Date
 189     ${delta} =    DateTime.Subtract_Date_From_Date    ${date_leader_elected}    ${date_start}
 190     # TODO: Consider extracting the block which depends on ${heal_timeout} value into two keywords, and have just one If to decide which one to call.
 191     BuiltIn.Run_Keyword_If    ${heal_timeout}==${HEAL_WITHIN_TRANS_TIMEOUT} and ${delta} > ${TRANSACTION_TIMEOUT}    BuiltIn.Fail    We are not able to heal the cluster within transaction timout because new leader election took more time.
 192     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 193     BuiltIn.Should_Be_Equal    ${resp}    ${NONE}    No response expected, received ${resp}
 194     BuiltIn.Sleep    ${heal_timeout}
 195     KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${leader}
 196     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 197     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 198     BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 199     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${shard_name}!!    ${shard_type}    verify_restconf=False
 200     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 201     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${TRANSACTION_TIMEOUT}    Leader_Isolation_Heal_Within_Tt
 202     ...    ELSE    Prefix_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 203     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 204     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 205
 206 Leader_Isolation_Heal_Within_Tt
 207     [Documentation]    The leader isolation test case end if the heal happens within transaction timeout. All write transaction
 208     ...    producers shoudl finish without error.
 209     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 210     : FOR    ${resp}    IN    @{resp_list}
 211     \    TemplatedRequests.Check_Status_Code    ${resp}
 212
 213 Module_Leader_Isolation_Heal_Default
 214     [Arguments]    ${isolated_node}    ${time_to_finish}
 215     [Documentation]    The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
 216     ...    Then all write transaction producers should finish without error.
 217     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 218     BuiltIn.Log    ${resp}
 219     # TODO: check on response status code
 220     ${restart_producer_node_idx_as_list}    BuiltIn.Create_List    ${isolated_node}
 221     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 222     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
 223     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
 224     ...    reset_globals=${False}
 225     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 226     : FOR    ${resp}    IN    @{resp_list}
 227     \    TemplatedRequests.Check_Status_Code    ${resp}
 228
 229 Prefix_Leader_Isolation_Heal_Default
 230     [Arguments]    ${isolated_node}    ${time_to_finish}
 231     [Documentation]    The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
 232     ...    Then all write transaction producers shoudl finish without error.
 233     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 234     BuiltIn.Log    ${resp}
 235     # TODO: check on response status code
 236     ${restart_producer_node_idx_as_list}    BuiltIn.Create_List    ${isolated_node}
 237     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 238     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
 239     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    reset_globals=${False}
 240     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 241     : FOR    ${resp}    IN    @{resp_list}
 242     \    TemplatedRequests.Check_Status_Code    ${resp}
 243
 244 Client_Isolation_Test_Templ
 245     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 246     [Documentation]    Implements client isolation test scenario.
 247     ${all_indices} =    ClusterManagement.List_All_Indices
 248     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 249     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 250     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 251     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 252     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 253     ${client_node_idx_as_list}    BuiltIn.Create_List    ${client_node_dst}
 254     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${client_node_ip_as_list}    ${client_node_idx_as_list}    ${ID_PREFIX}    ${DURATION_90S}    ${TRANSACTION_RATE_1K}    chained_flag=${trans_chain_flag}
 255     BuiltIn.Sleep    5s
 256     ${start_date}    DateTime.Get_Current_Date
 257     ${timeout_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${TRANSACTION_TIMEOUT}
 258     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${HARD_TIMEOUT}
 259     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${client_node_dst}
 260     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 261     ${trans_timeout} =    Get_Seconds_To_Time    ${timeout_date}
 262     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${trans_timeout}    1s    Ongoing_Transactions_Not_Failed_Yet
 263     ${hard_timeout} =    Get_Seconds_To_Time    ${abort_date}
 264     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${hard_timeout}    1s    Ongoing_Transactions_Not_Failed_Yet
 265     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    Ongoing_Transactions_Failed
 266     [Teardown]    BuiltIn.Run Keywords    KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${client_node_dst}
 267     ...    AND    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 268     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 269     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 270
 271 Client_Isolation_PrefBasedShard_Test_Templ
 272     [Arguments]    ${listener_node_role}    ${isolated_transactions_flag}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 273     [Documentation]    Implements client isolation test scenario.
 274     ${all_indices} =    ClusterManagement.List_All_Indices
 275     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 276     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 277     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 278     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 279     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 280     ${client_node_idx_as_list}    BuiltIn.Create_List    ${client_node_dst}
 281     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${client_node_ip_as_list}    ${client_node_idx_as_list}    ${ID_PREFIX}    ${DURATION_90S}    ${TRANSACTION_RATE_1K}    isolated_transactions_flag=${isolated_transactions_flag}
 282     BuiltIn.Sleep    5s
 283     ${start_date}    DateTime.Get_Current_Date
 284     ${timeout_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${TRANSACTION_TIMEOUT}
 285     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${HARD_TIMEOUT}
 286     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${client_node_dst}
 287     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 288     ${trans_timeout} =    Get_Seconds_To_Time    ${timeout_date}
 289     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${trans_timeout}    1s    Ongoing_Transactions_Not_Failed_Yet
 290     ${hard_timeout} =    Get_Seconds_To_Time    ${abort_date}
 291     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${hard_timeout}    1s    Ongoing_Transactions_Not_Failed_Yet
 292     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    Ongoing_Transactions_Failed
 293     [Teardown]    BuiltIn.Run Keywords    KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${client_node_dst}
 294     ...    AND    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 295     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 296     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 297
 298 Ongoing_Transactions_Not_Failed_Yet
 299     [Documentation]    Verify that no write-transaction rpc finished, means they are still running.
 300     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 301     BuiltIn.Should_Be_Equal    ${None}    ${resp}    ${resp} not expected.
 302
 303 Ongoing_Transactions_Failed
 304     [Documentation]    Verify if write-transaction failed.
 305     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 306     Check_Status_Code    ${resp}    explicit_status_codes=${TRANSACTION_FAILED}
 307
 308 Get_Seconds_To_Time
 309     [Arguments]    ${date_in_future}
 310     [Documentation]    Return number of seconds remaining to ${date_in_future}.
 311     ${date_now} =    DateTime.Get_Current_Date
 312     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
 313     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 314
 315 Remote_Listener_Test_Templ
 316     [Arguments]    ${listener_node_role}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 317     [Documentation]    Implements remote listener test scenario.
 318     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 319     ${all_indices} =    ClusterManagement.List_All_Indices
 320     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 321     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 322     ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 323     MdsalLowlevel.Subscribe_Dtcl    ${listener_node_dst}
 324     ${subscribed} =    BuiltIn.Set_Variable    ${True}
 325     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 326     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${DURATION_10S}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
 327     BuiltIn.Sleep    5s
 328     ClusterAdmin.Remove_Shard_Replica    ${listener_node_dst}    ${shard_name}    member-${listener_node_dst}    ${shard_type}
 329     ${alive_replicas} =    ClusterManagement.List_Indices_Minus_Member    ${listener_node_dst}    ${all_indices}
 330     ${newleader}    ${newfollower_list} =    BuiltIn.Wait_Until_Keyword_Succeeds    45s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}
 331     ...    shard_type=${shard_type}    member_index_list=${alive_replicas}    verify_restconf=False
 332     BuiltIn.Run_Keyword_If    "${listener_node_role}" != "leader"    BuiltIn.Should_Be_Equal_As_Numbers    ${leader}    ${newleader}
 333     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 334     : FOR    ${resp}    IN    @{resp_list}
 335     \    TemplatedRequests.Check_Status_Code    ${resp}
 336     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}
 337     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 338     BuiltIn.Should_Be_True    ${copy_matches}
 339     [Teardown]    BuiltIn.Run_Keywords    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}
 340     ...    AND    ClusterAdmin.Add_Shard_Replica    ${listener_node_dst}    ${shard_name}    ${shard_type}
 341     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}
 342     ...    verify_restconf=False    shard_type=${shard_type}
 343
 344 Remote_Listener_PrefBasedShard_Test_Templ
 345     [Arguments]    ${listener_node_role}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 346     [Documentation]    Implements listener isolation test scenario.
 347     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 348     ${all_indices} =    ClusterManagement.List_All_Indices
 349     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 350     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 351     ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 352     MdsalLowlevel.Subscribe_Ddtl    ${listener_node_dst}
 353     ${subscribed} =    BuiltIn.Set_Variable    ${True}
 354     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 355     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${DURATION_10S}    ${TRANSACTION_RATE_1K}
 356     BuiltIn.Sleep    5s
 357     ClusterAdmin.Remove_Prefix_Shard_Replica    ${listener_node_dst}    ${shard_name}    member-${listener_node_dst}    ${shard_type}
 358     ${alive_replicas} =    ClusterManagement.List_Indices_Minus_Member    ${listener_node_dst}    ${all_indices}
 359     ${newleader}    ${newfollower_list} =    BuiltIn.Wait_Until_Keyword_Succeeds    45s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!
 360     ...    member_index_list=${alive_replicas}    verify_restconf=False    shard_type=${shard_type}
 361     BuiltIn.Run_Keyword_If    "${listener_node_role}" != "leader"    BuiltIn.Should_Be_Equal_As_Numbers    ${leader}    ${newleader}
 362     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 363     : FOR    ${resp}    IN    @{resp_list}
 364     \    TemplatedRequests.Check_Status_Code    ${resp}
 365     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Ddtl    ${listener_node_dst}
 366     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 367     BuiltIn.Should_Be_True    ${copy_matches}
 368     [Teardown]    BuiltIn.Run_Keywords    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Ddtl    ${listener_node_dst}
 369     ...    AND    ClusterAdmin.Add_Prefix_Shard_Replica    ${listener_node_dst}    ${shard_name}    ${shard_type}
 370     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!
 371     ...    verify_restconf=False    shard_type=${shard_type}
 372
 373 Create_Prefix_Based_Shard_And_Verify
 374     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 375     [Documentation]    Create prefix based shard with replicas on all nodes
 376     ${all_indices} =    ClusterManagement.List_All_Indices
 377     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 378     MdsalLowlevel.Create_Prefix_Shard    ${node_to_trigger}    ${prefix}    ${all_indices}
 379     BuiltIn.Wait_Until_Keyword_Succeeds    30s    3s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${prefix}!!    shard_type=${SHARD_TYPE}    member_index_list=${all_indices}
 380     ...    verify_restconf=False
 381
 382 Remove_Prefix_Based_Shard_And_Verify
 383     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 384     [Documentation]    Remove prefix based shard with all replicas
 385     ${all_indices} =    ClusterManagement.List_All_Indices
 386     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 387     MdsalLowlevel.Remove_Prefix_Shard    ${node_to_trigger}    ${prefix}
 388     : FOR    ${idx}    IN    @{all_indices}
 389     \    BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    Verify_Shard_Replica_Removed    ${idx}    ${prefix}!!
 390     \    ...    ${SHARD_TYPE}
 391
 392 Verify_Shard_Replica_Removed
 393     [Arguments]    ${member_index}    ${shard_name}    ${shard_type}
 394     [Documentation]    Verify that shard is removed. Jolokia return 404 for shard memeber.
 395     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 396     ${type_class} =    Resolve_Shard_Type_Class    shard_type=${shard_type}
 397     ${uri} =    BuiltIn.Set_Variable    /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
 398     ${text}    TemplatedRequests.Get_From_Uri    uri=${uri}    session=${session}
 399     BuiltIn.Should_Contain    ${text}    "status":404    javax.management.InstanceNotFoundException
 400
 401 Restart_Test_Templ
 402     [Documentation]    Kill every odl node and start again.
 403     ClusterManagement.Kill_Members_From_List_Or_All
 404     ClusterManagement.Clean_Directories_On_List_Or_All    tmp_dir=/tmp
 405     ClusterManagement.Start_Members_From_List_Or_All
 406     BuiltIn.Wait_Until_Keyword_Succeeds    300s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}    verify_restconf=True
 407     ClusterManagement.Run_Bash_Command_On_List_Or_All    ps -ef | grep java