csit/suites/controller/dom_rpc_broker/action_provider_partition_and_heal.robot

   1 *** Settings ***
   2 Documentation     DOMRpcBroker testing: RPC Action Provider Partition And Heal
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This tests establishes that the RPC service for actions operates correctly
  11 ...               when faced with node failures.
  12 ...               This suite supports more than three node cluster setup too.
  13 Suite Setup       Setup_Kw
  14 Suite Teardown    SSHLibrary.Close_All_Connections
  15 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  16 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
  17 Default Tags      critical
  18 Library           Collections
  19 Library           SSHLibrary
  20 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  21 Resource          ${CURDIR}/../../../libraries/MdsalLowlevel.robot
  22 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  23 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  24 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  25
  26 *** Variables ***
  27 @{INSTALLED_RPC_MEMEBER_IDX_LIST}    ${1}    ${2}
  28 ${TESTED_MEMBER_WITHOUT_RPC_IDX}    ${3}
  29 ${CONSTANT_PREFIX}    member-
  30 ${CONTEXT}        context
  31
  32 *** Test Cases ***
  33 Register_Rpc_On_Two_Nodes
  34     [Documentation]    Register rpc on two nodes of the odl cluster.
  35     : FOR    ${index}    IN    @{INSTALLED_RPC_MEMEBER_IDX_LIST}
  36     \    MdsalLowlevel.Register_Bound_Constant    ${index}    ${CONTEXT}    ${CONSTANT_PREFIX}${index}
  37
  38 Invoke_Rpc_On_Each_Node
  39     [Documentation]    Invoke get-contexted-constant rpc on every node of the cluster. When requested on the node with
  40     ...    local instance the local value is expected. If invoked on the node with no local instance, any remote
  41     ...    value is expected. From the constant returned from the ${TESTED_MEMBER_WITHOUT_RPC_IDX} node (with no rpc instance) an index of
  42     ...    the node to be isolated is derived. And in the tc Invoke_Rpc_On_Remaining_Nodes a different constant
  43     ...    is expected. The second for loop makes the suite suitable for more that 3 nodes cluster.
  44     : FOR    ${index}    IN    @{INSTALLED_RPC_MEMEBER_IDX_LIST}
  45     \    Verify_Local_Rpc_Invoked    ${index}
  46     : FOR    ${index}    IN    @{non_installed_rpc_member_idx_list}
  47     \    ${constant} =    Verify_Any_Remote_Rpc_Invoked    ${index}
  48     \    BuiltIn.Run_Keyword_If    "${index}" == "${TESTED_MEMBER_WITHOUT_RPC_IDX}"    BuiltIn.Set_Suite_Variable    ${initial_const_on_tested_non_rpc_member}    ${constant}
  49     ${isolated_idx} =    String.Replace_String    ${initial_const_on_tested_non_rpc_member}    ${CONSTANT_PREFIX}    ${EMPTY}
  50     BuiltIn.Set_Suite_Variable    ${isolated_idx}    ${${isolated_idx}}
  51
  52 Isolate_One_Node
  53     [Documentation]    Isolate one node with registered rpc.
  54     ClusterManagement.Isolate_Member_From_List_Or_All    ${isolated_idx}
  55
  56 Invoke_Rpc_On_Remaining_Nodes
  57     [Documentation]    Invoke rpc on non-islolated nodes. As the only instance of rpc remained in the non-isolated
  58     ...    cluster nodes, only this value is expected.
  59     ${index_list} =    ClusterManagement.List_Indices_Minus_Member    ${isolated_idx}    ${all_indices}
  60     : FOR    ${index}    IN    @{index_list}
  61     \    ${constant} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Any_Remote_Rpc_Invoked    ${index}
  62     \    BuiltIn.Should_Not_Be_Equal_As_Strings    ${CONSTANT_PREFIX}${isolated_idx}    ${constant}
  63
  64 Rejoin_Isolated_Member
  65     [Documentation]    Rejoin isolated node
  66     ClusterManagement.Rejoin_Member_From_List_Or_All    ${isolated_idx}
  67
  68 Invoke_Rpc_On_Each_Node_Again
  69     [Documentation]    Invoke rpc get-contexted-constant on every node. When requested on the node with
  70     ...    local instance the local value is expected. If invoked on the node with no local instance, any remote
  71     ...    value is expected.
  72     : FOR    ${index}    IN    @{all_indices}
  73     \    BuiltIn.Run_Keyword_If    ${index} in ${INSTALLED_RPC_MEMEBER_IDX_LIST}    Verify_Local_Rpc_Invoked    ${index}
  74     \    BuiltIn.Run_Keyword_Unless    ${index} in ${INSTALLED_RPC_MEMEBER_IDX_LIST}    WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    20s    3s    Verify_Any_Remote_Rpc_Invoked
  75     \    ...    ${index}
  76
  77 Unregister_Rpc_On_Each_Node
  78     [Documentation]    Inregister rpc on both nodes.
  79     : FOR    ${index}    IN    @{INSTALLED_RPC_MEMEBER_IDX_LIST}
  80     \    MdsalLowlevel.Unregister_Bound_Constant    ${index}    ${CONTEXT}
  81
  82 *** Keywords ***
  83 Setup_Kw
  84     [Documentation]    Setup keyword. Create ${possible_constants} list with possible variables of remote constants.
  85     SetupUtils.Setup_Utils_For_Setup_And_Teardown    http_timeout=30
  86     ${all_indices} =    ClusterManagement.List_All_Indices
  87     BuiltIn.Set_Suite_Variable    ${all_indices}
  88     ${non_installed_rpc_member_idx_list} =    ClusterManagement.List_All_Indices
  89     ${possible_constants} =    BuiltIn.Create_List
  90     : FOR    ${index}    IN    @{INSTALLED_RPC_MEMEBER_IDX_LIST}
  91     \    Collections.Append_To_List    ${possible_constants}    ${CONSTANT_PREFIX}${index}
  92     \    ${non_installed_rpc_member_idx_list} =    ClusterManagement.List_Indices_Minus_Member    ${index}    ${non_installed_rpc_member_idx_list}
  93     BuiltIn.Set_Suite_Variable    ${possible_constants}
  94     BuiltIn.Set_Suite_Variable    ${non_installed_rpc_member_idx_list}
  95
  96 Verify_Local_Rpc_Invoked
  97     [Arguments]    ${member_index}
  98     [Documentation]    Verify that local constant is received.
  99     ${constant} =    MdsalLowlevel.Get_Contexted_Constant    ${member_index}    ${CONTEXT}
 100     BuiltIn.Should_Be_Equal_As_Strings    ${CONSTANT_PREFIX}${member_index}    ${constant}
 101     BuiltIn.Return_From_Keyword    ${constant}
 102
 103 Verify_Any_Remote_Rpc_Invoked
 104     [Arguments]    ${member_index}
 105     [Documentation]    Verify that any valid constant is received.
 106     ${constant} =    MdsalLowlevel.Get_Contexted_Constant    ${member_index}    ${CONTEXT}
 107     Collections.List_Should_Contain_Value    ${possible_constants}    ${constant}
 108     BuiltIn.Return_From_Keyword    ${constant}