csit/suites/controller/dom_rpc_broker/rpc_provider_partition_and_heal.robot

   1 *** Settings ***
   2 Documentation     DOMRpcBroker testing: RPC Provider Partition And Heal
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This tests establishes that the RPC service operates correctly when faced
  11 ...               with node failures.
  12 ...               This suite supports more than three node cluster setup too.
  13 Suite Setup       Setup_Kw
  14 Suite Teardown    SSHLibrary.Close_All_Connections
  15 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  16 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
  17 Default Tags      critical
  18 Library           SSHLibrary
  19 Resource          ${CURDIR}/../../../libraries/controller/DrbCommons.robot
  20 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  21 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  22
  23 *** Variables ***
  24 @{INSTALLED_RPC_MEMEBER_IDX_LIST}    ${1}    ${2}
  25 ${TESTED_MEMBER_WITHOUT_RPC_IDX}    ${3}
  26 @{NON_WORKING_RPC_STATUS_CODE}    ${501}
  27
  28 *** Test Cases ***
  29 Register_Rpc_On_Two_Nodes
  30     [Documentation]    Register rpc on two nodes of the odl cluster.
  31     DrbCommons.Register_Rpc_On_Nodes    ${INSTALLED_RPC_MEMEBER_IDX_LIST}
  32
  33 Invoke_Rpc_On_Each_Node
  34     [Documentation]    Invoke get-constant rpc on every node of the cluster. When requested on the node with
  35     ...    local instance the local value is expected. If invoked on the node with no local instance, any remote
  36     ...    value is expected.
  37     DrbCommons.Verify_Constant_On_Active_Nodes
  38
  39 Isolate_One_Node
  40     [Documentation]    Isolate one node with registered rpc.
  41     ...    From the constant returned from the ${TESTED_MEMBER_WITHOUT_RPC_IDX} node (with no rpc instance) an index of
  42     ...    the node to be isolated is derived. And in the tc Invoke_Rpc_On_Remaining_Nodes a different constant
  43     ...    is expected.
  44     ${isolated_idx} =    DrbCommons.Get_Constant_Index_From_Node    ${TESTED_MEMBER_WITHOUT_RPC_IDX}
  45     BuiltIn.Set_Suite_Variable    ${isolated_idx}
  46     DrbCommons.Isolate_Node    ${isolated_idx}
  47
  48 Invoke_Rpc_On_Isolated_Node
  49     [Documentation]    Invoke rpc on isolated node. Because rpc is registered on this node, local constant
  50     ...    is expected.
  51     BuiltIn.Wait_Until_Keyword_Succeeds    3x    2s    DrbCommons.Verify_Constant_On_Registered_Node    ${isolated_idx}
  52
  53 Invoke_Rpc_On_Remaining_Nodes
  54     [Documentation]    Invoke rpc on non-islolated nodes.
  55     DrbCommons.Verify_Constant_On_Active_Nodes
  56
  57 Rejoin_Isolated_Member
  58     [Documentation]    Rejoin isolated node
  59     DrbCommons.Rejoin_Node    ${isolated_idx}
  60
  61 Invoke_Rpc_On_Each_Node_Again
  62     [Documentation]    Invoke rpc get-constant on every node. When requested on the node with
  63     ...    local instance the local value is expected. If invoked on the node with no local instance, any remote
  64     ...    value is expected.
  65     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    20s    3s    DrbCommons.Verify_Constant_On_Active_Nodes
  66
  67 Isolate_Member_Without_Registered_Rpc
  68     [Documentation]    Isolate one node with unregistered rpc.
  69     DrbCommons.Isolate_Node    ${TESTED_MEMBER_WITHOUT_RPC_IDX}
  70
  71 Verify_Rpc_Fails_On_Isolated_Member_Without_Rpc
  72     [Documentation]    Rpc should fail as it is requested on isolated node without rpc instance.
  73     BuiltIn.Wait_Until_Keyword_Succeeds    15s    2s    MdsalLowlevel.Get_Constant    ${TESTED_MEMBER_WITHOUT_RPC_IDX}    explicit_status_codes=${NON_WORKING_RPC_STATUS_CODE}
  74
  75 Rejoin_Isolated_Member_Without_Registered_Rpc
  76     [Documentation]    Rejoin isolated node.
  77     DrbCommons.Rejoin_Node    ${TESTED_MEMBER_WITHOUT_RPC_IDX}
  78
  79 Verify_Rpc_Again_Passes_On_Member_Without_Rpc
  80     [Documentation]    Verify rpc works after the node rejoin.
  81     BuiltIn.Wait_Until_Keyword_Succeeds    10x    3s    DrbCommons.Verify_Constant_On_Unregistered_Node    ${TESTED_MEMBER_WITHOUT_RPC_IDX}
  82
  83 Unregister_Rpc_On_Each_Node
  84     [Documentation]    Inregister rpc on both nodes.
  85     DrbCommons.Unregister_Rpc_On_Nodes    ${INSTALLED_RPC_MEMEBER_IDX_LIST}
  86
  87 *** Keywords ***
  88 Setup_Kw
  89     [Documentation]    Setup keyword. Create ${possible_constants} list with possible variables of remote constants.
  90     SetupUtils.Setup_Utils_For_Setup_And_Teardown    http_timeout=30
  91     DrbCommons.DrbCommons_Init