csit/suites/controller/singleton_service/global_rpc_isolate.robot

   1 *** Settings ***
   2 Documentation     Controller functional HA testing of global singleton rpcs.
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 Suite Setup       Setup_Suite
  10 Suite Teardown    SSHLibrary.Close_All_Connections
  11 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  12 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
  13 Default Tags      critical
  14 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  15 Resource          ${CURDIR}/../../../libraries/KarafKeywords.robot
  16 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  17 Resource          ${CURDIR}/../../../libraries/ShardStability.robot
  18 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  19 Resource          ${CURDIR}/../../../variables/Variables.robot
  20 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  21
  22 *** Variables ***
  23 ${RPC_URL}        /restconf/operations/basic-rpc-test:basic-global
  24 &{EMPTY_DICT}
  25 ${SERVICE}        Basic-rpc-test']
  26 ${TEST_LOG_LEVEL}    info
  27 @{TEST_LOG_COMPONENTS}    org.opendaylight.controller.remote.rpc
  28 ${EOS_URL}        /restconf/operational/entity-owners:entity-owners
  29 ${RPC_STATUS_ISOLATED}    501
  30 @{NO_TAGS}
  31
  32 *** Test Cases ***
  33 Get_Basic_Rpc_Test_Owner
  34     [Documentation]    Find a service owner and successors.
  35     [Tags]    @{NO_TAGS}
  36     Get_Present_Brt_Owner_And_Successors    1    store=${True}
  37
  38 Rpc_Before_Isolation_On_Owner
  39     [Documentation]    Run rpc on the service owner.
  40     Run_Rpc    ${brt_owner}
  41
  42 Rpc_Before_Isolation_On_Successors
  43     [Documentation]    Run rpc on non owher cluster nodes.
  44     : FOR    ${idx}    IN    @{brt_successors}
  45     \    Run_Rpc    ${idx}
  46
  47 Isolate_Current_Owner_Member
  48     [Documentation]    Isolating cluster node which is the owner.
  49     [Tags]    @{NO_TAGS}
  50     ClusterManagement.Isolate_Member_From_List_Or_All    ${brt_owner}
  51     BuiltIn.Set Suite variable    ${old_brt_owner}    ${brt_owner}
  52     BuiltIn.Set Suite variable    ${old_brt_successors}    ${brt_successors}
  53
  54 Verify_New_Basic_Rpc_Test_Owner_Elected
  55     [Documentation]    Verify new owner of the service is elected.
  56     ${idx}=    Collections.Get_From_List    ${old_brt_successors}    0
  57     BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}    member_index_list=${old_brt_successors}
  58     BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    Verify_Owner_Elected    ${True}    ${old_brt_owner}    ${idx}
  59     Get_Present_Brt_Owner_And_Successors    ${idx}    store=${True}
  60
  61 Rpc_On_Isolated_Node
  62     [Documentation]    Run rpc on isolated cluster node.
  63     ${session} =    Resolve_Http_Session_For_Member    member_index=${old_brt_owner}
  64     BuiltIn.Run_Keyword_And_Ignore_Error    Get_And_Log_EOS_Output_To_Karaf_Log    ${session}
  65     BuiltIn.Pass_Execution    Rpc on isolated node may work for some time(bug 8207), then will fail (bug 8214)
  66     ${resp} =    RequestsLibrary.Post Request    ${session}    ${RPC_URL}    data=${EMPTY}
  67     BuiltIn.Should_Be_Equal_As_Numbers    ${resp.status_code}    ${RPC_STATUS_ISOLATED}
  68
  69 Rpc_On_Non_Isolated_Cluster_Nodes
  70     [Documentation]    Run rpc on remained cluster nodes.
  71     : FOR    ${idx}    IN    @{old_brt_successors}
  72     \    BuiltIn.Wait_Until_Keyword_Succeeds    60s    3s    Run_Rpc    ${idx}
  73
  74 Rejoin_Isolated_Member
  75     [Documentation]    Rejoin isolated node
  76     [Tags]    @{NO_TAGS}
  77     ClusterManagement.Rejoin_Member_From_List_Or_All    ${old_brt_owner}
  78     BuiltIn.Wait_Until_Keyword_Succeeds    60s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}
  79
  80 Verify_New_Owner_Remained_After_Rejoin
  81     [Documentation]    Verify no owner change happened after rejoin.
  82     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    15s    2s    Verify_Owner_Elected    ${False}    ${brt_owner}    ${brt_owner}
  83
  84 Rpc_After_Rejoin_On_New_Owner
  85     [Documentation]    Run rpc on the new service owner node.
  86     Run_Rpc    ${brt_owner}
  87
  88 Rpc_After_Rejoin_On_Old_Owner
  89     [Documentation]    Run rpc on rejoined cluster node.
  90     Run_Rpc    ${old_brt_owner}
  91
  92 Rpc_After_Rejoin_On_All
  93     [Documentation]    Run rpc again on all nodes.
  94     Run_Rpc    ${brt_owner}
  95     : FOR    ${idx}    IN    @{brt_successors}
  96     \    Run_Rpc    ${idx}
  97
  98 *** Keywords ***
  99 Setup_Suite
 100     SetupUtils.Setup_Utils_For_Setup_And_Teardown    http_timeout=30
 101     SetupUtils.Setup_Logging_For_Debug_Purposes_On_List_Or_All    ${TEST_LOG_LEVEL}    ${TEST_LOG_COMPONENTS}
 102
 103 Run_Rpc
 104     [Arguments]    ${node_idx}
 105     [Documentation]    Run rpc and log the entity ownership service details to karaf log.
 106     ...    Logging the details was a developer's request during the implementation to improve debugging.
 107     ${session} =    Resolve_Http_Session_For_Member    member_index=${node_idx}
 108     Get_And_Log_EOS_Output_To_Karaf_Log    ${session}
 109     TemplatedRequests.Post_To_Uri    ${RPC_URL}    ${EMPTY}    ${EMPTY_DICT}    ${EMPTY_DICT}    session=${session}
 110
 111 Verify_Owner_Elected
 112     [Arguments]    ${new_elected}    ${old_owner}    ${node_to_ask}
 113     [Documentation]    Verify new owner was elected or remained the same.
 114     ${owner}    ${successors}=    Get_Present_Brt_Owner_And_Successors    ${node_to_ask}
 115     BuiltIn.Run_Keyword_If    ${new_elected}    BuiltIn.Should_Not_Be_Equal_As_Numbers    ${old_owner}    ${owner}
 116     BuiltIn.Run_Keyword_Unless    ${new_elected}    BuiltIn.Should_Be_Equal_As_numbers    ${old_owner}    ${owner}
 117
 118 Get_Present_Brt_Owner_And_Successors
 119     [Arguments]    ${node_to_ask}    ${store}=${False}
 120     [Documentation]    Find a basic rpc test service owner and successors and store them if indicated.
 121     ${brt_owner}    ${brt_successors}=    ClusterManagement.Get_Owner_And_Successors_For_Device    ${SERVICE}    org.opendaylight.mdsal.ServiceEntityType    ${node_to_ask}
 122     BuiltIn.Run_Keyword_If    ${store}    BuiltIn.Set_Suite_Variable    ${brt_owner}    ${brt_owner}
 123     BuiltIn.Run_Keyword_If    ${store}    BuiltIn.Set_Suite_Variable    ${brt_successors}    ${brt_successors}
 124     BuiltIn.Return_From_Keyword    ${brt_owner}    ${brt_successors}
 125
 126 Get_And_Log_EOS_Output_To_Karaf_Log
 127     [Arguments]    ${session}
 128     [Documentation]    Log the entity ownership service details to karaf.log
 129     ${out} =    TemplatedRequests.Get_From_Uri    ${EOS_URL}    session=${session}
 130     KarafKeywords.Log_Message_To_Controller_Karaf    EOS rest resp: ${out}