csit/suites/mdsal/netty_replicate/netty-replicate.robot

   1 *** Settings ***
   2 Documentation     Test suite for testing MD-SAL netty replication functionality
   3 Suite Setup       Setup_Suite
   4 Test Setup        Setup_Test
   5 Test Teardown     Teardown_Test
   6 Default Tags      3node    critical    netty-replicate
   7 Library           SSHLibrary
   8 Resource          ${CURDIR}/../../../libraries/NettyReplication.robot
   9 Resource          ${CURDIR}/../../../libraries/KarafKeywords.robot
  10 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  11 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  12 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  13 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  14 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  15 Resource          ${CURDIR}/../../../libraries/Utils.robot
  16
  17 *** Variables ***
  18 ${CARPEOPLE_DEV_FOLDER}    ${CURDIR}/../../../variables/carpeople/crud
  19 ${ADDITIONAL_SOURCE_NODE_INDEX}    ${3}
  20 @{MULTIPLE_SINK_NODES_INDEXES}    ${2}    ${3}
  21
  22 *** Test Cases ***
  23 Replicate_Config_Addition
  24     [Documentation]    Adding new configuration on source node should be replicated on sink node.
  25     NettyReplication.Setup_Netty_Replication
  26     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${5}
  27
  28 Replicate_Config_Update
  29     [Documentation]    Updating existing configuration on source node should be replicated on sink node.
  30     NettyReplication.Setup_Netty_Replication
  31     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${5}    iter_j_offset=${0}
  32     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${5}    iter_j_offset=${5}
  33
  34 Replicte_Config_Deletion
  35     [Documentation]    Updating existing configuration on source node should be replicated on sink node.
  36     NettyReplication.Setup_Netty_Replication
  37     &{mapping} =    BuiltIn.Create_Dictionary
  38     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${5}
  39     Delete_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people
  40
  41 Replicate_Multiple_Changes_to_Config
  42     [Documentation]    CRUD configuration changes done on source node should be replicated on sink node.
  43     NettyReplication.Setup_Netty_Replication
  44     &{mapping_1} =    BuiltIn.Create_Dictionary
  45     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${5}    iter_j_offset=${0}
  46     &{mapping_2} =    BuiltIn.Create_Dictionary
  47     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${7}    iter_j_offset=${0}
  48     &{mapping_2_updated} =    BuiltIn.Create_Dictionary
  49     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${7}    iter_j_offset=${2}
  50     Delete_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people
  51
  52 Replicate_Multiple_Changes_to_Multiple_Sinks
  53     [Documentation]    CRUD configuration changes done on source node should be replicated on multiple sink nodes.
  54     NettyReplication.Setup_Netty_Replication    source_memeber_index=${DEFAULT_NETTY_SOURCE_NODE_INDEX}    sink_members_indexes=@{MULTIPLE_SINK_NODES_INDEXES}
  55     &{mapping_1} =    BuiltIn.Create_Dictionary
  56     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    sink_node_indexes=@{MULTIPLE_SINK_NODES_INDEXES}    iterations=${5}    iter_j_offset=${0}
  57     &{mapping_2} =    BuiltIn.Create_Dictionary
  58     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    sink_node_indexes=@{MULTIPLE_SINK_NODES_INDEXES}    iterations=${7}    iter_j_offset=${0}
  59     &{mapping_2_updated} =    BuiltIn.Create_Dictionary
  60     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    sink_node_indexes=@{MULTIPLE_SINK_NODES_INDEXES}    iterations=${8}    iter_j_offset=${2}
  61     Delete_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people
  62
  63 Sink_Catch_Up_To_Changes_After_Opening_Connection
  64     [Documentation]    After opening sink part of connection, sink should catch up to changes made on the source node, which were made before opening connection.
  65     NettyReplication.Open_Source_Connection
  66     &{mapping} =    BuiltIn.Create_Dictionary
  67     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    sink_node_indexes=@{EMPTY}    iterations=${5}
  68     ${netty_sink_session_alias} =    Resolve_Http_Session_For_Member    member_index=@{DEFAULT_NETTY_SINK_NODE_INDEXES}[0]
  69     Verify_Config_Is_Not_Present    ${CARPEOPLE_DEV_FOLDER}/people    session=${netty_sink_session_alias}    wait_time=5s
  70     NettyReplication.Open_Sink_Connection
  71     Verify_Config_Is_Present    ${CARPEOPLE_DEV_FOLDER}/people    session=${netty_sink_session_alias}    iterations=${5}
  72
  73 Reconnect_After_Lost_Connection
  74     [Documentation]    Test if sink sucessfuly reconnects after lost connection and if changes made during lost connection are present on reconnected sink's datastore.
  75     NettyReplication.Setup_Netty_Replication
  76     ClusterManagement.Isolate_Member_From_List_Or_All    @{DEFAULT_NETTY_SINK_NODE_INDEXES}[0]
  77     &{mapping} =    BuiltIn.Create_Dictionary
  78     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    sink_node_indexes=@{EMPTY}    iterations=${5}
  79     ${netty_sink_session_alias} =    Resolve_Http_Session_For_Member    member_index=@{DEFAULT_NETTY_SINK_NODE_INDEXES}[0]
  80     Verify_Config_Is_Not_Present    ${CARPEOPLE_DEV_FOLDER}/people    session=${netty_sink_session_alias}    wait_time=5s
  81     ClusterManagement.Rejoin_Member_From_List_Or_All    @{DEFAULT_NETTY_SINK_NODE_INDEXES}[0]
  82     Verify_Config_Is_Present    ${CARPEOPLE_DEV_FOLDER}/people    session=${netty_sink_session_alias}    iterations=${5}
  83     [Teardown]    Teardown_Isolation_Test
  84
  85 Change_Replication_Source
  86     [Documentation]    After changing replication source ODL to another ODL, sink should reflect datastore from newer source, forgeting changes from the old datastore.
  87     NettyReplication.Setup_Netty_Replication
  88     &{mapping_older} =    BuiltIn.Create_Dictionary
  89     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/people    iterations=${5}
  90     NettyReplication.Teardown_Netty_Replication
  91     # switch source node 1 -> 3
  92     NettyReplication.Setup_Netty_Replication    source_memeber_index=${ADDITIONAL_SOURCE_NODE_INDEX}    sink_members_indexes=${DEFAULT_NETTY_SINK_NODE_INDEXES}
  93     &{mapping_newer} =    BuiltIn.Create_Dictionary
  94     Put_Config_And_Verify    ${CARPEOPLE_DEV_FOLDER}/cars    source_node_index=${ADDITIONAL_SOURCE_NODE_INDEX}    iterations=${8}
  95     # check if data from old source was forgotten (removed)
  96     ${netty_sink_session_alias} =    Resolve_Http_Session_For_Member    member_index=@{DEFAULT_NETTY_SINK_NODE_INDEXES}[0]
  97     Verify_Config_Is_Not_Present    ${CARPEOPLE_DEV_FOLDER}/people    session=${netty_sink_session_alias}    removal=True
  98
  99 *** Keywords ***
 100 Setup_suite
 101     [Documentation]    Open karaf connections to each ODL and deconfigure cluster to prevent interferance of shard replication with netty replication.
 102     KarafKeywords.Setup_Karaf_Keywords
 103     # deconfigure cluster
 104     ${members_index_list} =    List_Indices_Or_All
 105     FOR    ${cluster_member_index}    IN    @{members_index_list}
 106         ${member_ip_address} =    ClusterManagement.Resolve_Ip_Address_For_Member    ${cluster_member_index}
 107         # backup old cluster configuration files
 108         Run_Bash_Command_On_Member    command=pushd ${karaf_home} && tar -cvf /tmp/config_backup.tar ./configuration/initial/ && popd    member_index=${cluster_member_index}
 109         Run_Bash_Command_On_Member    command=pushd ${karaf_home} && ./bin/configure_cluster.sh 1 ${member_ip_address} && popd    member_index=${cluster_member_index}
 110     END
 111
 112 Setup_Test
 113     [Documentation]    Clear data and set loging.
 114     Clear_Data_On_All_Nodes_And_Restart
 115     SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 116
 117 Teardown_Test
 118     [Documentation]    Close all netty replication connections and show linked bugs in case of failure.
 119     FOR    ${sink_member_index}    IN    @{ClusterManagement__member_index_list}
 120         NettyReplication.Close_Source_Connection    ${sink_member_index}
 121         NettyReplication.Close_Sink_Connection    ${sink_member_index}
 122     END
 123     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 124
 125 Teardown_Isolation_Test
 126     [Documentation]    Tear down test when isolation is used. Additionally to traditional teardown resets iptables rules set during isolation.
 127     FOR    ${member_index}    IN    @{ClusterManagement__member_index_list}
 128         ClusterManagement.Rejoin_Member_From_List_Or_All    ${member_index}
 129     END
 130     Teardown_Test
 131
 132 Clear_Data_On_All_Nodes_And_Restart
 133     [Documentation]    Clear data after stoping all members and then restart all members.
 134     ClusterManagement.Stop_Members_From_List_Or_All
 135     ClusterManagement.Clean_Journals_Data_And_Snapshots_On_List_Or_All
 136     ClusterManagement.Start_Members_From_List_Or_All
 137
 138 Put_Config_And_Verify
 139     [Arguments]    ${template_folder}    ${mapping}={}    ${source_node_index}=${DEFAULT_NETTY_SOURCE_NODE_INDEX}    @{sink_node_indexes}=${DEFAULT_NETTY_SINK_NODE_INDEXES}
 140     ...    ${iterations}=${1}    ${iter_j_offset}=${0}
 141     [Documentation]    Request put config on netty replicate source and verify changes has been made on both source and sinks.
 142     ${netty_source_session_alias} =    Resolve_Http_Session_For_Member    member_index=${source_node_index}
 143     TemplatedRequests.Put_As_Json_Templated    ${template_folder}    session=${netty_source_session_alias}    iterations=${iterations}    iter_j_offset=${iter_j_offset}
 144     Verify_Config_Is_Present    template_folder=${template_folder}    mapping=${mapping}    session=${netty_source_session_alias}    iterations=${iterations}    iter_j_offset=${iter_j_offset}
 145     FOR    ${sink_node_index}    IN    @{sink_node_indexes}
 146         ${netty_sink_session_alias} =    Resolve_Http_Session_For_Member    member_index=${sink_node_index}
 147         Verify_Config_Is_Present    template_folder=${template_folder}    mapping=${mapping}    session=${netty_sink_session_alias}    iterations=${iterations}    iter_j_offset=${iter_j_offset}
 148     END
 149
 150 Verify_Config_Is_Present
 151     [Arguments]    ${template_folder}    ${session}    ${mapping}={}    ${iterations}=${1}    ${iter_j_offset}=${0}
 152     [Documentation]    Verify config is present on target node datastore by using templated request.
 153     BuiltIn.Should_Not_Be_Empty    ${session}    Could not verify, session to node is not opened
 154     BuiltIn.Wait_Until_Keyword_Succeeds    10x    3s    TemplatedRequests.Get_As_Json_Templated    ${template_folder}
 155     ...    verify=True    session=${session}    iterations=${iterations}    iter_j_offset=${iter_j_offset}
 156
 157 Delete_Config_And_Verify
 158     [Arguments]    ${template_folder}    ${mapping}={}    ${source_node_index}=${DEFAULT_NETTY_SOURCE_NODE_INDEX}    @{sink_node_indexes}=${DEFAULT_NETTY_SINK_NODE_INDEXES}
 159     ...    ${iterations}=${1}
 160     [Documentation]    Request delete config on netty replicate source and verify changes has been made on both source and sink.
 161     ${netty_source_session_alias} =    Resolve_Http_Session_For_Member    member_index=${source_node_index}
 162     TemplatedRequests.Delete_Templated    ${template_folder}    session=${netty_source_session_alias}
 163     Verify_Config_Is_Not_Present    template_folder=${template_folder}    mapping=${mapping}    session=${netty_source_session_alias}    removal=True
 164     FOR    ${sink_node_index}    IN    @{sink_node_indexes}
 165         ${netty_sink_session_alias} =    Resolve_Http_Session_For_Member    member_index=${sink_node_index}
 166         Verify_Config_Is_Not_Present    template_folder=${template_folder}    mapping=${mapping}    session=${netty_sink_session_alias}    removal=True
 167     END
 168
 169 Verify_Config_Is_Not_Present
 170     [Arguments]    ${template_folder}    ${session}    ${mapping}={}    ${removal}=False    ${wait_time}=0s
 171     [Documentation]    Verify config is not present on the target node datastore by using templated request. Should get return code 404 or 409.
 172     ...    removal - Retries until config is not present (for cases of cofig deletion when config migh be present at begining, but disapears later)
 173     ...    wait_time - Repeatedly checks for specific amount of time if config does not appear (for cases of config addition when config is not present, but might appear after some time)
 174     BuiltIn.Should_Not_Be_Empty    ${session}    Could not verify, session to node is not opened
 175     ${uri} =    TemplatedRequests.Resolve_Text_From_Template_Folder    folder=${template_folder}    base_name=location    extension=uri    mapping=${mapping}
 176     BuiltIn.Run_Keyword_If    ${removal}    Until_Confg_Is_Removed    ${session}    ${uri}
 177     ...    ELSE IF    "${wait_time}" != "0s"    Config_Does_Not_Appear_During_Time    ${session}    ${uri}    ${wait_time}
 178     ...    ELSE    Utils.No_Content_From_URI    ${session}    ${uri}
 179
 180 Until_Confg_Is_Removed
 181     [Arguments]    ${session}    ${uri}
 182     [Documentation]    Retry until config is not avaibale
 183     BuiltIn.Wait_Until_Keyword_Succeeds    10x    3s    Utils.No_Content_From_URI    ${session}    ${uri}
 184
 185 Config_Does_Not_Appear_During_Time
 186     [Arguments]    ${session}    ${uri}    ${wait_time}
 187     [Documentation]    Repeatedly check if config does not appear during the wait time
 188     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${wait_time}    1s    Utils.No_Content_From_URI    ${session}    ${uri}