csit/suites/controller/singleton_service/global_rpc_isolate.robot

   1 *** Settings ***
   2 Documentation       Controller functional HA testing of global singleton rpcs.
   3 ...
   4 ...                 Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9
  10 Resource            ${CURDIR}/../../../libraries/ClusterManagement.robot
  11 Resource            ${CURDIR}/../../../libraries/KarafKeywords.robot
  12 Resource            ${CURDIR}/../../../libraries/SetupUtils.robot
  13 Resource            ${CURDIR}/../../../libraries/ShardStability.robot
  14 Resource            ${CURDIR}/../../../libraries/TemplatedRequests.robot
  15 Resource            ${CURDIR}/../../../variables/Variables.robot
  16 Resource            ${CURDIR}/../../../libraries/WaitForFailure.robot
  17
  18 Suite Setup         Setup_Suite
  19 Suite Teardown      SSHLibrary.Close_All_Connections
  20 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  21 Test Teardown       SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
  22
  23 Default Tags        critical
  24
  25
  26 *** Variables ***
  27 ${RPC_URL}                  /restconf/operations/basic-rpc-test:basic-global
  28 &{EMPTY_DICT}
  29 ${SERVICE}                  Basic-rpc-test']
  30 ${TEST_LOG_LEVEL}           info
  31 @{TEST_LOG_COMPONENTS}      org.opendaylight.controller.remote.rpc
  32 ${EOS_URL}                  /restconf/operational/entity-owners:entity-owners
  33 ${RPC_STATUS_ISOLATED}      501
  34
  35
  36 *** Test Cases ***
  37 Get_Basic_Rpc_Test_Owner
  38     [Documentation]    Find a service owner and successors.
  39     Get_Present_Brt_Owner_And_Successors    1    store=${True}
  40
  41 Rpc_Before_Isolation_On_Owner
  42     [Documentation]    Run rpc on the service owner.
  43     Run_Rpc    ${brt_owner}
  44
  45 Rpc_Before_Isolation_On_Successors
  46     [Documentation]    Run rpc on non owher cluster nodes.
  47     FOR    ${idx}    IN    @{brt_successors}
  48         Run_Rpc    ${idx}
  49     END
  50
  51 Isolate_Current_Owner_Member
  52     [Documentation]    Isolating cluster node which is the owner.
  53     ClusterManagement.Isolate_Member_From_List_Or_All    ${brt_owner}
  54     BuiltIn.Set Suite variable    ${old_brt_owner}    ${brt_owner}
  55     BuiltIn.Set Suite variable    ${old_brt_successors}    ${brt_successors}
  56
  57 Verify_New_Basic_Rpc_Test_Owner_Elected
  58     [Documentation]    Verify new owner of the service is elected.
  59     ${idx} =    Collections.Get_From_List    ${old_brt_successors}    0
  60     BuiltIn.Wait_Until_Keyword_Succeeds
  61     ...    70s
  62     ...    10s
  63     ...    ShardStability.Shards_Stability_Get_Details
  64     ...    ${DEFAULT_SHARD_LIST}
  65     ...    member_index_list=${old_brt_successors}
  66     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Owner_Elected    ${True}    ${old_brt_owner}    ${idx}
  67     Get_Present_Brt_Owner_And_Successors    ${idx}    store=${True}
  68
  69 Rpc_On_Isolated_Node
  70     [Documentation]    Run rpc on isolated cluster node.
  71     ${session} =    Resolve_Http_Session_For_Member    member_index=${old_brt_owner}
  72     BuiltIn.Run_Keyword_And_Ignore_Error    Get_And_Log_EOS_Output_To_Karaf_Log    ${session}
  73     BuiltIn.Pass_Execution    Rpc on isolated node may work for some time(bug 8207), then will fail (bug 8214)
  74     ${resp} =    RequestsLibrary.Post Request    ${session}    ${RPC_URL}    data=${EMPTY}
  75     BuiltIn.Should_Be_Equal_As_Numbers    ${resp.status_code}    ${RPC_STATUS_ISOLATED}
  76
  77 Rpc_On_Non_Isolated_Cluster_Nodes
  78     [Documentation]    Run rpc on remained cluster nodes.
  79     FOR    ${idx}    IN    @{old_brt_successors}
  80         BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Run_Rpc    ${idx}
  81     END
  82
  83 Rejoin_Isolated_Member
  84     [Documentation]    Rejoin isolated node
  85     ClusterManagement.Rejoin_Member_From_List_Or_All    ${old_brt_owner}
  86     BuiltIn.Wait_Until_Keyword_Succeeds
  87     ...    70s
  88     ...    10s
  89     ...    ShardStability.Shards_Stability_Get_Details
  90     ...    ${DEFAULT_SHARD_LIST}
  91
  92 Rpc_After_Rejoin_On_New_Owner
  93     [Documentation]    Run rpc on the new service owner node.
  94     Run_Rpc    ${brt_owner}
  95
  96 Rpc_After_Rejoin_On_Old_Owner
  97     [Documentation]    Run rpc on rejoined cluster node.
  98     Run_Rpc    ${old_brt_owner}
  99
 100 Rpc_After_Rejoin_On_All
 101     [Documentation]    Run rpc again on all nodes.
 102     Run_Rpc    ${brt_owner}
 103     FOR    ${idx}    IN    @{brt_successors}
 104         Run_Rpc    ${idx}
 105     END
 106
 107
 108 *** Keywords ***
 109 Setup_Suite
 110     [Documentation]    FIXME: Add a documentation.
 111     SetupUtils.Setup_Utils_For_Setup_And_Teardown    http_timeout=30
 112     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
 113
 114 Run_Rpc
 115     [Documentation]    Run rpc and log the entity ownership service details to karaf log.
 116     ...    Logging the details was a developer's request during the implementation to improve debugging.
 117     [Arguments]    ${node_idx}
 118     ${session} =    Resolve_Http_Session_For_Member    member_index=${node_idx}
 119     Get_And_Log_EOS_Output_To_Karaf_Log    ${session}
 120     TemplatedRequests.Post_To_Uri    ${RPC_URL}    ${EMPTY}    ${HEADERS_XML}    ${ACCEPT_XML}    session=${session}
 121
 122 Verify_Owner_Elected
 123     [Documentation]    Verify new owner was elected or remained the same.
 124     [Arguments]    ${new_elected}    ${old_owner}    ${node_to_ask}
 125     ${owner}    ${successors} =    Get_Present_Brt_Owner_And_Successors    ${node_to_ask}
 126     IF    ${new_elected}
 127         BuiltIn.Should_Not_Be_Equal_As_Numbers    ${old_owner}    ${owner}
 128     END
 129     IF    not ${new_elected}
 130         BuiltIn.Should_Be_Equal_As_numbers    ${old_owner}    ${owner}
 131     END
 132
 133 Get_Present_Brt_Owner_And_Successors
 134     [Documentation]    Find a basic rpc test service owner and successors and store them if indicated.
 135     [Arguments]    ${node_to_ask}    ${store}=${False}
 136     ${brt_owner}    ${brt_successors} =    ClusterManagement.Get_Owner_And_Successors_For_Device
 137     ...    ${SERVICE}
 138     ...    org.opendaylight.mdsal.ServiceEntityType
 139     ...    ${node_to_ask}
 140     IF    ${store}    BuiltIn.Set_Suite_Variable    ${brt_owner}    ${brt_owner}
 141     IF    ${store}
 142         BuiltIn.Set_Suite_Variable    ${brt_successors}    ${brt_successors}
 143     END
 144     RETURN    ${brt_owner}    ${brt_successors}
 145
 146 Get_And_Log_EOS_Output_To_Karaf_Log
 147     [Documentation]    Log the entity ownership service details to karaf.log
 148     [Arguments]    ${session}
 149     ${out} =    TemplatedRequests.Get_From_Uri    ${EOS_URL}    session=${session}
 150     KarafKeywords.Log_Message_To_Controller_Karaf    EOS rest resp: ${out}