csit/libraries/controller/DdbCommons.robot

   1 *** Settings ***
   2 Documentation     DOMDataBroker testing: Common keywords
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This resource file implements various test cases templates.
  11 ...               FIXME: add a link to a document (when published) where the scenarios are defined
  12 ...
  13 ...               TODO: When checking first response in isolation scenarior, make sure it comes from the expected member.
  14 Library           ${CURDIR}/../MdsalLowlevelPy.py
  15 Resource          ${CURDIR}/../ClusterAdmin.robot
  16 Resource          ${CURDIR}/../ClusterManagement.robot
  17 Resource          ${CURDIR}/../KarafKeywords.robot
  18 Resource          ${CURDIR}/../MdsalLowlevel.robot
  19 Resource          ${CURDIR}/../TemplatedRequests.robot
  20 Resource          ${CURDIR}/../ShardStability.robot
  21 Resource          ${CURDIR}/../WaitForFailure.robot
  22
  23 *** Variables ***
  24 ${SHARD_NAME}     default
  25 ${SHARD_TYPE}     config
  26 ${TRANSACTION_RATE_1K}    ${1000}
  27 ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}    ${2*${REQUEST_TIMEOUT}}
  28 ${TRANSACTION_PRODUCTION_TIME}    ${40}
  29 ${SLEEP_AFTER_TRANSACTIONS_INIT}    5s
  30 ${ID_PREFIX}      prefix-
  31 ${ID_PREFIX2}     prefix-    # different-prefix- has been used before, but currently is neither needed nor supported
  32 ${SIMPLE_TX}      ${False}
  33 ${CHAINED_TX}     ${True}
  34 ${ISOLATED_TRANS_TRUE}    ${True}
  35 ${ISOLATED_TRANS_FALSE}    ${False}
  36 ${JAVA_INTERNAL_RECONNECT_TIMEOUT}    ${30}
  37 ${REQUEST_TIMEOUT}    ${120}
  38 ${HEAL_WITHIN_REQUEST_TIMEOUT}    ${${JAVA_INTERNAL_RECONNECT_TIMEOUT}+10}
  39 ${HEAL_AFTER_REQUEST_TIMEOUT}    ${${REQUEST_TIMEOUT}+10}
  40 @{TRANSACTION_FAILED}    ${500}
  41 ${PREF_BASED_SHARD}    id-ints
  42 ${TEST_LOG_LEVEL}    info
  43 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller.cluster.sharding    org.opendaylight.controller.cluster.datastore
  44 ${HEAL_WITHIN_TRANS_TIMEOUT}    ${0}
  45
  46 *** Keywords ***
  47 Explicit_Leader_Movement_Test_Templ
  48     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  49     [Documentation]    Implements explicit leader movement test scenario.
  50     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}
  51     ...    ${shard_type}
  52     KarafKeywords.Log_Message_To_Controller_Karaf    Starting leader movement from node${idx_from} to node${idx_to}, transaction producer at node${idx_trans}.
  53     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  54     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  55     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${ip_trans_as_list}    ${idx_trans_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
  56     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
  57     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
  58     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}
  59     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
  60     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  61     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  62     Check_Status_Of_First_Response    ${resp_list}
  63
  64 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
  65     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
  66     [Documentation]    Implements explicit leader movement test scenario.
  67     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}!!
  68     ...    ${shard_type}
  69     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  70     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  71     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${ip_trans_as_list}    ${idx_trans_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}
  72     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
  73     MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}
  74     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!
  75     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
  76     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  77     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  78     Check_Status_Of_First_Response    ${resp_list}
  79
  80 Get_Node_Indexes_For_The_ELM_Test
  81     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}    ${shard_type}
  82     [Documentation]    Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
  83     ...    producer should be deployed.
  84     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    verify_restconf=False
  85     ${idx_from} =    BuiltIn.Set_Variable    ${leader}
  86     ${idx_to} =    BuiltIn.Set_Variable    @{follower_list}[0]
  87     ${idx_trans} =    BuiltIn.Set_Variable_If    "${leader_from}" == "remote" and "${leader_to}" == "remote"    @{follower_list}[1]    "${leader_from}" == "local"    ${leader}    "${leader_to}" == "local"
  88     ...    @{follower_list}[0]
  89     BuiltIn.Return_From_Keyword    ${idx_from}    ${idx_to}    ${idx_trans}
  90
  91 Clean_Leader_Shutdown_Test_Templ
  92     [Arguments]    ${leader_location}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  93     [Documentation]    Implements clean leader shutdown test scenario.
  94     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}    ${shard_name}    ${shard_type}
  95     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
  96     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
  97     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
  98     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
  99     BuiltIn.Comment    Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
 100     ClusterAdmin.Remove_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
 101     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${actual_leader}    ${shard_name}    ${shard_type}
 102     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 103     Check_Status_Of_First_Response    ${resp_list}
 104     [Teardown]    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    120s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}
 105     ...    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
 106
 107 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
 108     [Arguments]    ${leader_location}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 109     [Documentation]    Implements clean leader shutdown test scenario.
 110     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}    ${shard_name}!!    ${shard_type}
 111     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
 112     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
 113     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}
 114     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 115     BuiltIn.Comment    Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
 116     ClusterAdmin.Remove_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
 117     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${actual_leader}    ${shard_name}!!    ${shard_type}
 118     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 119     Check_Status_Of_First_Response    ${resp_list}
 120     [Teardown]    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    120s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!
 121     ...    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
 122
 123 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
 124     [Arguments]    ${leader_location}    ${shard_name}    ${shard_type}
 125     [Documentation]    Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
 126     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    verify_restconf=False
 127     ${follower_list_leangth} =    BuiltIn.Evaluate    ${NUM_ODL_SYSTEM}-1
 128     BuiltIn.Length_Should_Be    ${follower_list}    ${follower_list_leangth}
 129     ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    @{follower_list}[0]
 130     BuiltIn.Return_From_Keyword    ${producer_idx}    ${leader}    ${follower_list}
 131
 132 Leader_Isolation_Test_Templ
 133     [Arguments]    ${heal_timeout}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 134     [Documentation]    Implements leader isolation test scenario.
 135     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 136     ${producing_transactions_time} =    BuiltIn.Set_Variable_If    ${heal_timeout}<${REQUEST_TIMEOUT}    ${${REQUEST_TIMEOUT}+60}    ${2*${REQUEST_TIMEOUT}}
 137     ${all_indices} =    ClusterManagement.List_All_Indices
 138     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 139     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 140     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
 141     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 142     ${date_start} =    DateTime.Get_Current_Date
 143     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 144     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${leader}
 145     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 146     ${li_isolated}    BuiltIn.Set_Variable    ${True}
 147     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}    ${shard_type}    ${True}
 148     ...    ${leader}    member_index_list=${follower_list}    verify_restconf=False
 149     ${heal_date} =    DateTime.Add_Time_To_Date    ${date_start}    ${heal_timeout}
 150     ${sleep_to_heal} =    Get_Seconds_To_Time    ${heal_date}
 151     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 152     BuiltIn.Should_Be_Equal    ${resp}    ${NONE}    No response expected, received ${resp}
 153     BuiltIn.Sleep    ${sleep_to_heal}
 154     KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${leader}
 155     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 156     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 157     BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 158     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${shard_name}    ${shard_type}    verify_restconf=False
 159     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 160     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${REQUEST_TIMEOUT}    Leader_Isolation_Heal_Within_Rt
 161     ...    ELSE    Module_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 162     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 163     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 164
 165 Leader_Isolation_PrefBasedShard_Test_Templ
 166     [Arguments]    ${heal_timeout}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 167     [Documentation]    Implements leader isolation test scenario.
 168     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 169     ${producing_transactions_time} =    BuiltIn.Set_Variable_If    ${heal_timeout}<${REQUEST_TIMEOUT}    ${${REQUEST_TIMEOUT}+60}    ${2*${REQUEST_TIMEOUT}}
 170     ${all_indices} =    ClusterManagement.List_All_Indices
 171     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 172     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 173     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}
 174     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 175     ${date_start} =    DateTime.Get_Current_Date
 176     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 177     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${leader}
 178     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 179     ${li_isolated}    BuiltIn.Set_Variable    ${True}
 180     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!    ${shard_type}    ${True}
 181     ...    ${leader}    member_index_list=${follower_list}
 182     ${heal_date} =    DateTime.Add_Time_To_Date    ${date_start}    ${heal_timeout}
 183     ${sleep_to_heal} =    Get_Seconds_To_Time    ${heal_date}
 184     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 185     BuiltIn.Should_Be_Equal    ${resp}    ${NONE}    No response expected, received ${resp}
 186     BuiltIn.Sleep    ${sleep_to_heal}
 187     KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${leader}
 188     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 189     ${li_isolated}    BuiltIn.Set_Variable    ${False}
 190     BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 191     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${shard_name}!!    ${shard_type}    verify_restconf=False
 192     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 193     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${REQUEST_TIMEOUT}    Leader_Isolation_Heal_Within_Rt
 194     ...    ELSE    Prefix_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 195     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 196     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 197
 198 Leader_Isolation_Heal_Within_Rt
 199     [Documentation]    The leader isolation test case end if the heal happens within transaction timeout. All write transaction
 200     ...    producers shoudl finish without error.
 201     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 202     : FOR    ${resp}    IN    @{resp_list}
 203     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
 204
 205 Module_Leader_Isolation_Heal_Default
 206     [Arguments]    ${isolated_node}    ${time_to_finish}
 207     [Documentation]    The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
 208     ...    Then all write transaction producers should finish without error.
 209     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 210     BuiltIn.Log    ${resp}
 211     BuiltIn.Should_Not_Be_Equal    ${NONE}    ${resp}    Write-transaction should have returned error from isolated leader node.
 212     # TODO: check on response status code
 213     ${restart_producer_node_idx_as_list}    BuiltIn.Create_List    ${isolated_node}
 214     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 215     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
 216     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX2}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
 217     ...    reset_globals=${False}
 218     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 219     : FOR    ${resp}    IN    @{resp_list}
 220     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
 221
 222 Prefix_Leader_Isolation_Heal_Default
 223     [Arguments]    ${isolated_node}    ${time_to_finish}
 224     [Documentation]    The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
 225     ...    Then all write transaction producers shoudl finish without error.
 226     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 227     BuiltIn.Log    ${resp}
 228     BuiltIn.Should_Not_Be_Equal    ${NONE}    ${resp}    Produce-transaction should have returned error from isolated leader node.
 229     # TODO: check on response status code
 230     ${restart_producer_node_idx_as_list}    BuiltIn.Create_List    ${isolated_node}
 231     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 232     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
 233     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX2}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    reset_globals=${False}
 234     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 235     : FOR    ${resp}    IN    @{resp_list}
 236     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
 237
 238 Client_Isolation_Test_Templ
 239     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 240     [Documentation]    Implements client isolation test scenario.
 241     ${all_indices} =    ClusterManagement.List_All_Indices
 242     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 243     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 244     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 245     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 246     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 247     ${client_node_idx_as_list}    BuiltIn.Create_List    ${client_node_dst}
 248     ${start_date}    DateTime.Get_Current_Date
 249     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${REQUEST_TIMEOUT}
 250     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${client_node_ip_as_list}    ${client_node_idx_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}    ${TRANSACTION_RATE_1K}    chained_flag=${trans_chain_flag}
 251     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 252     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${client_node_dst}
 253     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 254     ${rpc_timeout} =    Get_Seconds_To_Time    ${abort_date}
 255     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${rpc_timeout}    1s    Ongoing_Transactions_Not_Failed_Yet
 256     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    Ongoing_Transactions_Failed
 257     [Teardown]    BuiltIn.Run Keywords    KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${client_node_dst}
 258     ...    AND    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 259     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 260     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 261
 262 Client_Isolation_PrefBasedShard_Test_Templ
 263     [Arguments]    ${listener_node_role}    ${isolated_transactions_flag}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 264     [Documentation]    Implements client isolation test scenario.
 265     ${all_indices} =    ClusterManagement.List_All_Indices
 266     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
 267     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 268     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 269     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 270     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 271     ${client_node_idx_as_list}    BuiltIn.Create_List    ${client_node_dst}
 272     ${start_date}    DateTime.Get_Current_Date
 273     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${REQUEST_TIMEOUT}
 274     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${client_node_ip_as_list}    ${client_node_idx_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}    ${TRANSACTION_RATE_1K}    isolated_transactions_flag=${isolated_transactions_flag}
 275     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 276     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${client_node_dst}
 277     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 278     ${rpc_timeout} =    Get_Seconds_To_Time    ${abort_date}
 279     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${rpc_timeout}    1s    Ongoing_Transactions_Not_Failed_Yet
 280     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    Ongoing_Transactions_Failed
 281     [Teardown]    BuiltIn.Run Keywords    KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${client_node_dst}
 282     ...    AND    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 283     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 284     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 285
 286 Ongoing_Transactions_Not_Failed_Yet
 287     [Documentation]    Verify that no write-transaction rpc finished, means they are still running.
 288     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 289     BuiltIn.Should_Be_Equal    ${None}    ${resp}    ${resp} not expected.
 290
 291 Ongoing_Transactions_Failed
 292     [Documentation]    Verify if write-transaction failed.
 293     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 294     Check_Status_Code    @{resp}[2]    explicit_status_codes=${TRANSACTION_FAILED}
 295
 296 Get_Seconds_To_Time
 297     [Arguments]    ${date_in_future}
 298     [Documentation]    Return number of seconds remaining to ${date_in_future}.
 299     ${date_now} =    DateTime.Get_Current_Date
 300     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
 301     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 302
 303 Listener_Stability_Test_Templ
 304     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 305     [Documentation]    Implements listener stability test scenario for module-based shards.
 306     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 307     ${all_indices} =    ClusterManagement.List_All_Indices
 308     ${idx_from}    ${idx_to}    ${idx_listen} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}
 309     ...    ${shard_type}
 310     MdsalLowlevel.Subscribe_Dtcl    ${idx_listen}
 311     ${subscribed} =    BuiltIn.Set_Variable    ${True}
 312     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 313     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
 314     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 315     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
 316     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}
 317     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
 318     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
 319     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 320     : FOR    ${resp}    IN    @{resp_list}
 321     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
 322     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}
 323     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 324     BuiltIn.Should_Be_True    ${copy_matches}
 325     [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}
 326
 327 Listener_Stability_PrefBasedShard_Test_Templ
 328     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 329     [Documentation]    Implements listener stability test scenario for prefix-based shards.
 330     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 331     ${all_indices} =    ClusterManagement.List_All_Indices
 332     ${idx_from}    ${idx_to}    ${idx_listen} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}!!
 333     ...    ${shard_type}
 334     MdsalLowlevel.Subscribe_Ddtl    ${idx_listen}
 335     ${subscribed} =    BuiltIn.Set_Variable    ${True}
 336     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 337     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}
 338     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 339     MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}
 340     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!
 341     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
 342     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
 343     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 344     : FOR    ${resp}    IN    @{resp_list}
 345     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
 346     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}
 347     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 348     BuiltIn.Should_Be_True    ${copy_matches}
 349     [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}
 350
 351 Create_Prefix_Based_Shard_And_Verify
 352     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 353     [Documentation]    Create prefix based shard with replicas on all nodes
 354     ${all_indices} =    ClusterManagement.List_All_Indices
 355     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 356     MdsalLowlevel.Create_Prefix_Shard    ${node_to_trigger}    ${prefix}    ${all_indices}
 357     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${prefix}!!    shard_type=${SHARD_TYPE}    member_index_list=${all_indices}
 358     ...    verify_restconf=False
 359
 360 Remove_Prefix_Based_Shard_And_Verify
 361     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 362     [Documentation]    Remove prefix based shard with all replicas
 363     ${all_indices} =    ClusterManagement.List_All_Indices
 364     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 365     MdsalLowlevel.Remove_Prefix_Shard    ${node_to_trigger}    ${prefix}
 366     : FOR    ${idx}    IN    @{all_indices}
 367     \    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${idx}    ${prefix}!!
 368     \    ...    ${SHARD_TYPE}
 369
 370 Verify_Shard_Replica_Not_Present
 371     [Arguments]    ${member_index}    ${shard_name}    ${shard_type}
 372     [Documentation]    Verify that shard is removed. Jolokia return 404 for shard memeber.
 373     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 374     ${type_class} =    Resolve_Shard_Type_Class    shard_type=${shard_type}
 375     ${uri} =    BuiltIn.Set_Variable    /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
 376     ${text}    TemplatedRequests.Get_From_Uri    uri=${uri}    session=${session}
 377     BuiltIn.Should_Contain    ${text}    "status":404    javax.management.InstanceNotFoundException
 378
 379 Restart_Test_Templ
 380     [Documentation]    Kill every odl node and start again.
 381     ClusterManagement.Kill_Members_From_List_Or_All
 382     ClusterManagement.Clean_Directories_On_List_Or_All    tmp_dir=/tmp
 383     ClusterManagement.Start_Members_From_List_Or_All
 384     BuiltIn.Wait_Until_Keyword_Succeeds    300s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}    verify_restconf=True
 385     ClusterManagement.Run_Bash_Command_On_List_Or_All    ps -ef | grep java
 386
 387 Check_Status_Of_First_Response
 388     [Arguments]    ${resp_list}
 389     [Documentation]    Extract first item from the list, third item of the tuple and call TemplatedRequests to check the http status code.
 390     # @{resp_list}[0][2] does not work
 391     ${tuple} =    BuiltIn.Set_Variable    @{resp_list}[0]
 392     TemplatedRequests.Check_Status_Code    @{tuple}[2]
 393
 394 Change_Use_Tell_Based_Protocol
 395     [Arguments]    ${status}    ${DATASTORE_CFG}
 396     [Documentation]    Change status use-tell-based-protocol to True or False
 397     ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^#use-tell-based-protocol=true/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
 398     ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^#use-tell-based-protocol=false/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
 399     BuiltIn.Run_Keyword_And_Return_If    "${status}" == "True"    ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^use-tell-based-protocol=false/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
 400     BuiltIn.Run_Keyword_And_Return_If    "${status}" == "False"    ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^use-tell-based-protocol=true/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
 401     BuiltIn.Fail    Failure in status. Status can be True or False.