csit/libraries/controller/DdbCommons.robot

   1 *** Settings ***
   2 Documentation       DOMDataBroker testing: Common keywords
   3 ...
   4 ...                 Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...                 This resource file implements various test cases templates.
  11 ...                 FIXME: add a link to a document (when published) where the scenarios are defined
  12 ...
  13 ...                 TODO: When checking first response in isolation scenarior, make sure it comes from the expected member.
  14
  15 Library             ${CURDIR}/../MdsalLowlevelPy.py
  16 Resource            ${CURDIR}/../ClusterAdmin.robot
  17 Resource            ${CURDIR}/../ClusterManagement.robot
  18 Resource            ${CURDIR}/../KarafKeywords.robot
  19 Resource            ${CURDIR}/../MdsalLowlevel.robot
  20 Resource            ${CURDIR}/../TemplatedRequests.robot
  21 Resource            ${CURDIR}/../ShardStability.robot
  22 Resource            ${CURDIR}/../WaitForFailure.robot
  23
  24
  25 *** Variables ***
  26 ${SHARD_NAME}                                       default
  27 ${SHARD_TYPE}                                       config
  28 ${TRANSACTION_RATE_1K}                              ${1000}
  29 ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}       ${2*${REQUEST_TIMEOUT}}
  30 ${TRANSACTION_PRODUCTION_TIME}                      ${40}
  31 ${SLEEP_AFTER_TRANSACTIONS_INIT}                    5s
  32 ${ID_PREFIX}                                        prefix-
  33 # different-prefix- has been used before, but currently is neither needed nor supported
  34 ${ID_PREFIX2}
  35 ...                                                 prefix-
  36 ${SIMPLE_TX}                                        ${False}
  37 ${CHAINED_TX}                                       ${True}
  38 ${ISOLATED_TRANS_TRUE}                              ${True}
  39 ${ISOLATED_TRANS_FALSE}                             ${False}
  40 ${JAVA_INTERNAL_RECONNECT_TIMEOUT}                  ${30}
  41 ${REQUEST_TIMEOUT}                                  ${120}
  42 ${HEAL_WITHIN_REQUEST_TIMEOUT}                      ${${JAVA_INTERNAL_RECONNECT_TIMEOUT}+10}
  43 ${HEAL_AFTER_REQUEST_TIMEOUT}                       ${${REQUEST_TIMEOUT}+10}
  44 @{TRANSACTION_FAILED}                               ${500}
  45 ${PREF_BASED_SHARD}                                 id-ints
  46 ${TEST_LOG_LEVEL}                                   info
  47 @{TEST_LOG_COMPONENTS}
  48 ...                                                 org.opendaylight.controller.cluster.sharding
  49 ...                                                 org.opendaylight.controller.cluster.datastore
  50 ${HEAL_WITHIN_TRANS_TIMEOUT}                        ${0}
  51
  52
  53 *** Keywords ***
  54 Explicit_Leader_Movement_Test_Templ
  55     [Documentation]    Implements explicit leader movement test scenario.
  56     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  57     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test
  58     ...    ${leader_from}
  59     ...    ${leader_to}
  60     ...    ${shard_name}
  61     ...    ${shard_type}
  62     KarafKeywords.Log_Message_To_Controller_Karaf
  63     ...    Starting leader movement from node${idx_from} to node${idx_to}, transaction producer at node${idx_trans}.
  64     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  65     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  66     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
  67     ...    ${ip_trans_as_list}
  68     ...    ${idx_trans_as_list}
  69     ...    ${ID_PREFIX}
  70     ...    ${TRANSACTION_PRODUCTION_TIME}
  71     ...    ${TRANSACTION_RATE_1K}
  72     ...    chained_flag=${CHAINED_TX}
  73     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
  74     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
  75     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds
  76     ...    60s
  77     ...    5s
  78     ...    ClusterManagement.Verify_Shard_Leader_Elected
  79     ...    ${shard_name}
  80     ...    ${shard_type}
  81     ...    ${True}
  82     ...    ${idx_from}
  83     ...    verify_restconf=False
  84     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  85     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
  86     Check_Status_Of_First_Response    ${resp_list}
  87
  88 Explicit_Leader_Movement_PrefBasedShard_Test_Templ
  89     [Documentation]    Implements explicit leader movement test scenario.
  90     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
  91     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test
  92     ...    ${leader_from}
  93     ...    ${leader_to}
  94     ...    ${shard_name}!!
  95     ...    ${shard_type}
  96     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  97     ${idx_trans_as_list} =    BuiltIn.Create_List    ${idx_trans}
  98     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
  99     ...    ${ip_trans_as_list}
 100     ...    ${idx_trans_as_list}
 101     ...    ${ID_PREFIX}
 102     ...    ${TRANSACTION_PRODUCTION_TIME}
 103     ...    ${TRANSACTION_RATE_1K}
 104     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 105     MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}
 106     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds
 107     ...    60s
 108     ...    5s
 109     ...    ClusterManagement.Verify_Shard_Leader_Elected
 110     ...    ${shard_name}!!
 111     ...    ${shard_type}
 112     ...    ${True}
 113     ...    ${idx_from}
 114     ...    verify_restconf=False
 115     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
 116     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 117     Check_Status_Of_First_Response    ${resp_list}
 118
 119 Get_Node_Indexes_For_The_ELM_Test
 120     [Documentation]    Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
 121     ...    producer should be deployed.
 122     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}    ${shard_type}
 123     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard
 124     ...    shard_name=${shard_name}
 125     ...    shard_type=${shard_type}
 126     ...    verify_restconf=False
 127     ${idx_from} =    BuiltIn.Set_Variable    ${leader}
 128     ${idx_to} =    BuiltIn.Set_Variable    ${follower_list}[0]
 129     ${idx_trans} =    BuiltIn.Set_Variable_If
 130     ...    "${leader_from}" == "remote" and "${leader_to}" == "remote"
 131     ...    ${follower_list}[1]
 132     ...    "${leader_from}" == "local"
 133     ...    ${leader}
 134     ...    "${leader_to}" == "local"
 135     ...    ${follower_list}[0]
 136     RETURN    ${idx_from}    ${idx_to}    ${idx_trans}
 137
 138 Clean_Leader_Shutdown_Test_Templ
 139     [Documentation]    Implements clean leader shutdown test scenario.
 140     [Arguments]    ${leader_location}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 141     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
 142     ...    ${leader_location}
 143     ...    ${shard_name}
 144     ...    ${shard_type}
 145     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
 146     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
 147     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
 148     ...    ${producer_ip_as_list}
 149     ...    ${producer_idx_as_list}
 150     ...    ${ID_PREFIX}
 151     ...    ${TRANSACTION_PRODUCTION_TIME}
 152     ...    ${TRANSACTION_RATE_1K}
 153     ...    chained_flag=${CHAINED_TX}
 154     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 155     BuiltIn.Comment    Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
 156     ClusterAdmin.Remove_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
 157     BuiltIn.Wait_Until_Keyword_Succeeds
 158     ...    60s
 159     ...    5s
 160     ...    Verify_Shard_Replica_Not_Present
 161     ...    ${actual_leader}
 162     ...    ${shard_name}
 163     ...    ${shard_type}
 164     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 165     Check_Status_Of_First_Response    ${resp_list}
 166     [Teardown]    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    120s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}
 167     ...    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
 168
 169 Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
 170     [Documentation]    Implements clean leader shutdown test scenario.
 171     [Arguments]    ${leader_location}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 172     ${producer_idx}    ${actual_leader}    ${follower_list} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
 173     ...    ${leader_location}
 174     ...    ${shard_name}!!
 175     ...    ${shard_type}
 176     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
 177     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
 178     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
 179     ...    ${producer_ip_as_list}
 180     ...    ${producer_idx_as_list}
 181     ...    ${ID_PREFIX}
 182     ...    ${TRANSACTION_PRODUCTION_TIME}
 183     ...    ${TRANSACTION_RATE_1K}
 184     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 185     BuiltIn.Comment    Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
 186     ClusterAdmin.Remove_Prefix_Shard_Replica
 187     ...    ${actual_leader}
 188     ...    ${shard_name}
 189     ...    member-${actual_leader}
 190     ...    ${shard_type}
 191     BuiltIn.Wait_Until_Keyword_Succeeds
 192     ...    60s
 193     ...    5s
 194     ...    Verify_Shard_Replica_Not_Present
 195     ...    ${actual_leader}
 196     ...    ${shard_name}!!
 197     ...    ${shard_type}
 198     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 199     Check_Status_Of_First_Response    ${resp_list}
 200     [Teardown]    BuiltIn.Run_Keyword_And_Ignore_Error    BuiltIn.Wait_Until_Keyword_Succeeds    120s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!
 201     ...    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
 202
 203 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
 204     [Documentation]    Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
 205     [Arguments]    ${leader_location}    ${shard_name}    ${shard_type}
 206     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard
 207     ...    shard_name=${shard_name}
 208     ...    shard_type=${shard_type}
 209     ...    verify_restconf=False
 210     ${follower_list_leangth} =    BuiltIn.Evaluate    ${NUM_ODL_SYSTEM}-1
 211     BuiltIn.Length_Should_Be    ${follower_list}    ${follower_list_leangth}
 212     ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    ${follower_list}[0]
 213     RETURN    ${producer_idx}    ${leader}    ${follower_list}
 214
 215 Leader_Isolation_Test_Templ
 216     [Documentation]    Implements leader isolation test scenario.
 217     [Arguments]    ${heal_timeout}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 218     ${li_isolated} =    BuiltIn.Set_Variable    ${False}
 219     ${producing_transactions_time} =    BuiltIn.Set_Variable_If
 220     ...    ${heal_timeout}<${REQUEST_TIMEOUT}
 221     ...    ${${REQUEST_TIMEOUT}+60}
 222     ...    ${2*${REQUEST_TIMEOUT}}
 223     ${all_indices} =    ClusterManagement.List_All_Indices
 224     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard
 225     ...    shard_name=${shard_name}
 226     ...    shard_type=${shard_type}
 227     ...    member_index_list=${all_indices}
 228     ...    verify_restconf=False
 229     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 230     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
 231     ...    ${all_ip_list}
 232     ...    ${all_indices}
 233     ...    ${ID_PREFIX}
 234     ...    ${producing_transactions_time}
 235     ...    ${TRANSACTION_RATE_1K}
 236     ...    chained_flag=${CHAINED_TX}
 237     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 238     ${date_start} =    DateTime.Get_Current_Date
 239     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 240     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${leader}
 241     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 242     ${li_isolated} =    BuiltIn.Set_Variable    ${True}
 243     BuiltIn.Wait_Until_Keyword_Succeeds
 244     ...    60s
 245     ...    5s
 246     ...    ClusterManagement.Verify_Shard_Leader_Elected
 247     ...    ${shard_name}
 248     ...    ${shard_type}
 249     ...    ${True}
 250     ...    ${leader}
 251     ...    member_index_list=${follower_list}
 252     ...    verify_restconf=False
 253     ${heal_date} =    DateTime.Add_Time_To_Date    ${date_start}    ${heal_timeout}
 254     ${sleep_to_heal} =    Get_Seconds_To_Time    ${heal_date}
 255     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 256     BuiltIn.Should_Be_Equal    ${resp}    ${NONE}    No response expected, received ${resp}
 257     BuiltIn.Sleep    ${sleep_to_heal}
 258     KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${leader}
 259     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 260     ${li_isolated} =    BuiltIn.Set_Variable    ${False}
 261     BuiltIn.Wait_Until_Keyword_Succeeds
 262     ...    70s
 263     ...    10s
 264     ...    ShardStability.Shards_Stability_Get_Details
 265     ...    ${DEFAULT_SHARD_LIST}
 266     BuiltIn.Wait_Until_Keyword_Succeeds
 267     ...    60s
 268     ...    5s
 269     ...    ClusterManagement.Get_Leader_And_Followers_For_Shard
 270     ...    ${shard_name}
 271     ...    ${shard_type}
 272     ...    verify_restconf=False
 273     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 274     IF    ${heal_timeout} < ${REQUEST_TIMEOUT}
 275         Leader_Isolation_Heal_Within_Rt
 276     ELSE
 277         Module_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 278     END
 279     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 280     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 281
 282 Leader_Isolation_PrefBasedShard_Test_Templ
 283     [Documentation]    Implements leader isolation test scenario.
 284     [Arguments]    ${heal_timeout}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 285     ${li_isolated} =    BuiltIn.Set_Variable    ${False}
 286     ${producing_transactions_time} =    BuiltIn.Set_Variable_If
 287     ...    ${heal_timeout}<${REQUEST_TIMEOUT}
 288     ...    ${${REQUEST_TIMEOUT}+60}
 289     ...    ${2*${REQUEST_TIMEOUT}}
 290     ${all_indices} =    ClusterManagement.List_All_Indices
 291     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard
 292     ...    shard_name=${shard_name}!!
 293     ...    shard_type=${shard_type}
 294     ...    member_index_list=${all_indices}
 295     ...    verify_restconf=False
 296     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 297     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
 298     ...    ${all_ip_list}
 299     ...    ${all_indices}
 300     ...    ${ID_PREFIX}
 301     ...    ${producing_transactions_time}
 302     ...    ${TRANSACTION_RATE_1K}
 303     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 304     ${date_start} =    DateTime.Get_Current_Date
 305     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
 306     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${leader}
 307     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
 308     ${li_isolated} =    BuiltIn.Set_Variable    ${True}
 309     BuiltIn.Wait_Until_Keyword_Succeeds
 310     ...    60s
 311     ...    5s
 312     ...    ClusterManagement.Verify_Shard_Leader_Elected
 313     ...    ${shard_name}!!
 314     ...    ${shard_type}
 315     ...    ${True}
 316     ...    ${leader}
 317     ...    member_index_list=${follower_list}
 318     ...    verify_restconf=False
 319     ${heal_date} =    DateTime.Add_Time_To_Date    ${date_start}    ${heal_timeout}
 320     ${sleep_to_heal} =    Get_Seconds_To_Time    ${heal_date}
 321     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 322     BuiltIn.Should_Be_Equal    ${resp}    ${NONE}    No response expected, received ${resp}
 323     BuiltIn.Sleep    ${sleep_to_heal}
 324     KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${leader}
 325     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 326     ${li_isolated} =    BuiltIn.Set_Variable    ${False}
 327     BuiltIn.Wait_Until_Keyword_Succeeds
 328     ...    70s
 329     ...    10s
 330     ...    ShardStability.Shards_Stability_Get_Details
 331     ...    ${DEFAULT_SHARD_LIST}
 332     BuiltIn.Wait_Until_Keyword_Succeeds
 333     ...    60s
 334     ...    5s
 335     ...    ClusterManagement.Get_Leader_And_Followers_For_Shard
 336     ...    ${shard_name}!!
 337     ...    ${shard_type}
 338     ...    verify_restconf=False
 339     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
 340     IF    ${heal_timeout} < ${REQUEST_TIMEOUT}
 341         Leader_Isolation_Heal_Within_Rt
 342     ELSE
 343         Prefix_Leader_Isolation_Heal_Default    ${leader}    ${time_to_finish}
 344     END
 345     [Teardown]    BuiltIn.Run_Keyword_If    ${li_isolated}    BuiltIn.Run_Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
 346     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 347
 348 Leader_Isolation_Heal_Within_Rt
 349     [Documentation]    The leader isolation test case end if the heal happens within transaction timeout. All write transaction
 350     ...    producers shoudl finish without error.
 351     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 352     FOR    ${resp}    IN    @{resp_list}
 353         TemplatedRequests.Check_Status_Code    ${resp}[2]
 354     END
 355
 356 Module_Leader_Isolation_Heal_Default
 357     [Documentation]    The leader isolation test case end. The transaction producer on isolated node should fail and should be restarted.
 358     ...    Then all write transaction producers should finish without error.
 359     [Arguments]    ${isolated_node}    ${time_to_finish}
 360     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 361     BuiltIn.Log    ${resp}
 362     BuiltIn.Should_Not_Be_Equal
 363     ...    ${NONE}
 364     ...    ${resp}
 365     ...    Write-transaction should have returned error from isolated leader node.
 366     # TODO: check on response status code
 367     ${restart_producer_node_idx_as_list} =    BuiltIn.Create_List    ${isolated_node}
 368     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 369     ${restart_producer_node_ip_as_list} =    BuiltIn.Create_List    ${restart_producer_node_ip}
 370     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
 371     ...    ${restart_producer_node_ip_as_list}
 372     ...    ${restart_producer_node_idx_as_list}
 373     ...    ${ID_PREFIX2}
 374     ...    ${time_to_finish}
 375     ...    ${TRANSACTION_RATE_1K}
 376     ...    chained_flag=${CHAINED_TX}
 377     ...    reset_globals=${False}
 378     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 379     FOR    ${resp}    IN    @{resp_list}
 380         TemplatedRequests.Check_Status_Code    ${resp}[2]
 381     END
 382
 383 Prefix_Leader_Isolation_Heal_Default
 384     [Documentation]    The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
 385     ...    Then all write transaction producers shoudl finish without error.
 386     [Arguments]    ${isolated_node}    ${time_to_finish}
 387     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 388     BuiltIn.Log    ${resp}
 389     BuiltIn.Should_Not_Be_Equal
 390     ...    ${NONE}
 391     ...    ${resp}
 392     ...    Produce-transaction should have returned error from isolated leader node.
 393     # TODO: check on response status code
 394     ${restart_producer_node_idx_as_list} =    BuiltIn.Create_List    ${isolated_node}
 395     ${restart_producer_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${isolated_node}
 396     ${restart_producer_node_ip_as_list} =    BuiltIn.Create_List    ${restart_producer_node_ip}
 397     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
 398     ...    ${restart_producer_node_ip_as_list}
 399     ...    ${restart_producer_node_idx_as_list}
 400     ...    ${ID_PREFIX2}
 401     ...    ${time_to_finish}
 402     ...    ${TRANSACTION_RATE_1K}
 403     ...    reset_globals=${False}
 404     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 405     FOR    ${resp}    IN    @{resp_list}
 406         TemplatedRequests.Check_Status_Code    ${resp}[2]
 407     END
 408
 409 Client_Isolation_Test_Templ
 410     [Documentation]    Implements client isolation test scenario.
 411     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 412     ${all_indices} =    ClusterManagement.List_All_Indices
 413     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard
 414     ...    shard_name=${shard_name}
 415     ...    shard_type=${shard_type}
 416     ...    member_index_list=${all_indices}
 417     ...    verify_restconf=False
 418     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 419     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 420     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 421     ${client_node_ip_as_list} =    BuiltIn.Create_List    ${client_node_ip}
 422     ${client_node_idx_as_list} =    BuiltIn.Create_List    ${client_node_dst}
 423     ${start_date} =    DateTime.Get_Current_Date
 424     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${REQUEST_TIMEOUT}
 425     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
 426     ...    ${client_node_ip_as_list}
 427     ...    ${client_node_idx_as_list}
 428     ...    ${ID_PREFIX}
 429     ...    ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}
 430     ...    ${TRANSACTION_RATE_1K}
 431     ...    chained_flag=${trans_chain_flag}
 432     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 433     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${client_node_dst}
 434     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 435     ${rpc_timeout} =    Get_Seconds_To_Time    ${abort_date}
 436     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout
 437     ...    ${rpc_timeout}
 438     ...    1s
 439     ...    Ongoing_Transactions_Not_Failed_Yet
 440     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    Ongoing_Transactions_Failed
 441     [Teardown]    BuiltIn.Run Keywords    KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${client_node_dst}
 442     ...    AND    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 443     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 444     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 445
 446 Client_Isolation_PrefBasedShard_Test_Templ
 447     [Documentation]    Implements client isolation test scenario.
 448     [Arguments]    ${listener_node_role}    ${isolated_transactions_flag}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 449     ${all_indices} =    ClusterManagement.List_All_Indices
 450     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard
 451     ...    shard_name=${shard_name}!!
 452     ...    shard_type=${shard_type}
 453     ...    member_index_list=${all_indices}
 454     ...    verify_restconf=False
 455     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 456     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 457     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 458     ${client_node_ip_as_list} =    BuiltIn.Create_List    ${client_node_ip}
 459     ${client_node_idx_as_list} =    BuiltIn.Create_List    ${client_node_dst}
 460     ${start_date} =    DateTime.Get_Current_Date
 461     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${REQUEST_TIMEOUT}
 462     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
 463     ...    ${client_node_ip_as_list}
 464     ...    ${client_node_idx_as_list}
 465     ...    ${ID_PREFIX}
 466     ...    ${TRANSACTION_PRODUCTION_TIME_2X_REQ_TIMEOUT}
 467     ...    ${TRANSACTION_RATE_1K}
 468     ...    isolated_transactions_flag=${isolated_transactions_flag}
 469     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 470     KarafKeywords.Log_Message_To_Controller_Karaf    Isolating node ${client_node_dst}
 471     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 472     ${rpc_timeout} =    Get_Seconds_To_Time    ${abort_date}
 473     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout
 474     ...    ${rpc_timeout}
 475     ...    1s
 476     ...    Ongoing_Transactions_Not_Failed_Yet
 477     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    Ongoing_Transactions_Failed
 478     [Teardown]    BuiltIn.Run Keywords    KarafKeywords.Log_Message_To_Controller_Karaf    Rejoining node ${client_node_dst}
 479     ...    AND    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 480     ...    AND    BuiltIn.Wait_Until_Keyword_Succeeds    70s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
 481     ...    AND    MdsalLowlevelPy.Wait_For_Transactions
 482
 483 Ongoing_Transactions_Not_Failed_Yet
 484     [Documentation]    Verify that no write-transaction rpc finished, means they are still running.
 485     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 486     BuiltIn.Should_Be_Equal    ${None}    ${resp}    ${resp} not expected.
 487
 488 Ongoing_Transactions_Failed
 489     [Documentation]    Verify if write-transaction failed.
 490     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
 491     Check_Status_Code    ${resp}[2]    explicit_status_codes=${TRANSACTION_FAILED}
 492
 493 Get_Seconds_To_Time
 494     [Documentation]    Return number of seconds remaining to ${date_in_future}.
 495     [Arguments]    ${date_in_future}
 496     ${date_now} =    DateTime.Get_Current_Date
 497     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
 498     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 499
 500 Listener_Stability_Test_Templ
 501     [Documentation]    Implements listener stability test scenario for module-based shards.
 502     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 503     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 504     ${all_indices} =    ClusterManagement.List_All_Indices
 505     ${idx_from}    ${idx_to}    ${idx_listen} =    Get_Node_Indexes_For_The_ELM_Test
 506     ...    ${leader_from}
 507     ...    ${leader_to}
 508     ...    ${shard_name}
 509     ...    ${shard_type}
 510     MdsalLowlevel.Subscribe_Dtcl    ${idx_listen}
 511     ${subscribed} =    BuiltIn.Set_Variable    ${True}
 512     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 513     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes
 514     ...    ${all_ip_list}
 515     ...    ${all_indices}
 516     ...    ${ID_PREFIX}
 517     ...    ${TRANSACTION_PRODUCTION_TIME}
 518     ...    ${TRANSACTION_RATE_1K}
 519     ...    chained_flag=${CHAINED_TX}
 520     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 521     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
 522     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds
 523     ...    60s
 524     ...    5s
 525     ...    ClusterManagement.Verify_Shard_Leader_Elected
 526     ...    ${shard_name}
 527     ...    ${shard_type}
 528     ...    ${True}
 529     ...    ${idx_from}
 530     ...    verify_restconf=False
 531     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
 532     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 533     FOR    ${resp}    IN    @{resp_list}
 534         TemplatedRequests.Check_Status_Code    ${resp}[2]
 535     END
 536     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}
 537     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 538     BuiltIn.Should_Be_True    ${copy_matches}
 539     [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}
 540
 541 Listener_Stability_PrefBasedShard_Test_Templ
 542     [Documentation]    Implements listener stability test scenario for prefix-based shards.
 543     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
 544     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 545     ${all_indices} =    ClusterManagement.List_All_Indices
 546     ${idx_from}    ${idx_to}    ${idx_listen} =    Get_Node_Indexes_For_The_ELM_Test
 547     ...    ${leader_from}
 548     ...    ${leader_to}
 549     ...    ${shard_name}!!
 550     ...    ${shard_type}
 551     MdsalLowlevel.Subscribe_Ddtl    ${idx_listen}
 552     ${subscribed} =    BuiltIn.Set_Variable    ${True}
 553     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 554     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes
 555     ...    ${all_ip_list}
 556     ...    ${all_indices}
 557     ...    ${ID_PREFIX}
 558     ...    ${TRANSACTION_PRODUCTION_TIME}
 559     ...    ${TRANSACTION_RATE_1K}
 560     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
 561     MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}
 562     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds
 563     ...    60s
 564     ...    5s
 565     ...    ClusterManagement.Verify_Shard_Leader_Elected
 566     ...    ${shard_name}!!
 567     ...    ${shard_type}
 568     ...    ${True}
 569     ...    ${idx_from}
 570     ...    verify_restconf=False
 571     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
 572     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
 573     FOR    ${resp}    IN    @{resp_list}
 574         TemplatedRequests.Check_Status_Code    ${resp}[2]
 575     END
 576     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}
 577     ${subscribed} =    BuiltIn.Set_Variable    ${False}
 578     BuiltIn.Should_Be_True    ${copy_matches}
 579     [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}
 580
 581 Create_Prefix_Based_Shard_And_Verify
 582     [Documentation]    Create prefix based shard with replicas on all nodes
 583     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 584     ${all_indices} =    ClusterManagement.List_All_Indices
 585     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 586     MdsalLowlevel.Create_Prefix_Shard    ${node_to_trigger}    ${prefix}    ${all_indices}
 587     BuiltIn.Wait_Until_Keyword_Succeeds
 588     ...    60s
 589     ...    5s
 590     ...    ClusterManagement.Get_Leader_And_Followers_For_Shard
 591     ...    shard_name=${prefix}!!
 592     ...    shard_type=${SHARD_TYPE}
 593     ...    member_index_list=${all_indices}
 594     ...    verify_restconf=False
 595
 596 Remove_Prefix_Based_Shard_And_Verify
 597     [Documentation]    Remove prefix based shard with all replicas
 598     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 599     ${all_indices} =    ClusterManagement.List_All_Indices
 600     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
 601     MdsalLowlevel.Remove_Prefix_Shard    ${node_to_trigger}    ${prefix}
 602     FOR    ${idx}    IN    @{all_indices}
 603         BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${idx}    ${prefix}!!
 604         ...    ${SHARD_TYPE}
 605     END
 606
 607 Verify_Shard_Replica_Not_Present
 608     [Documentation]    Verify that shard is removed. Jolokia return 404 for shard memeber.
 609     [Arguments]    ${member_index}    ${shard_name}    ${shard_type}
 610     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
 611     ${type_class} =    Resolve_Shard_Type_Class    shard_type=${shard_type}
 612     ${uri} =    BuiltIn.Set_Variable
 613     ...    /jolokia/read/org.opendaylight.controller:Category=Shards,name=member-${member_index}-shard-${shard_name}-${shard_type},type=${type_class}
 614     ${text} =    TemplatedRequests.Get_From_Uri    uri=${uri}    session=${session}
 615     BuiltIn.Should_Contain    ${text}    "status":404    javax.management.InstanceNotFoundException
 616
 617 Restart_Test_Templ
 618     [Documentation]    Stop every odl node and start again.
 619     ClusterManagement.Stop_Members_From_List_Or_All
 620     ClusterManagement.Clean_Directories_On_List_Or_All    tmp_dir=/tmp
 621     ClusterManagement.Start_Members_From_List_Or_All
 622     BuiltIn.Wait_Until_Keyword_Succeeds
 623     ...    300s
 624     ...    10s
 625     ...    ShardStability.Shards_Stability_Get_Details
 626     ...    ${DEFAULT_SHARD_LIST}
 627     ...    verify_restconf=True
 628     ClusterManagement.Run_Bash_Command_On_List_Or_All    ps -ef | grep java
 629
 630 Check_Status_Of_First_Response
 631     [Documentation]    Extract first item from the list, third item of the tuple and call TemplatedRequests to check the http status code.
 632     [Arguments]    ${resp_list}
 633     # ${resp_list}[0][2] does not work
 634     ${tuple} =    BuiltIn.Set_Variable    ${resp_list}[0]
 635     TemplatedRequests.Check_Status_Code    ${tuple}[2]
 636
 637 Change_Use_Tell_Based_Protocol
 638     [Documentation]    Change status use-tell-based-protocol to True or False
 639     [Arguments]    ${status}    ${DATASTORE_CFG}
 640     ClusterManagement.Check_Bash_Command_On_List_Or_All
 641     ...    sed -ie "s/^#use-tell-based-protocol=true/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
 642     ClusterManagement.Check_Bash_Command_On_List_Or_All
 643     ...    sed -ie "s/^#use-tell-based-protocol=false/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
 644     BuiltIn.Run_Keyword_And_Return_If
 645     ...    "${status}" == "True"
 646     ...    ClusterManagement.Check_Bash_Command_On_List_Or_All
 647     ...    sed -ie "s/^use-tell-based-protocol=false/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
 648     BuiltIn.Run_Keyword_And_Return_If
 649     ...    "${status}" == "False"
 650     ...    ClusterManagement.Check_Bash_Command_On_List_Or_All
 651     ...    sed -ie "s/^use-tell-based-protocol=true/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
 652     BuiltIn.Fail    Failure in status. Status can be True or False.