csit/suites/netconf/clustering/entity-isolation-rejoin.robot

   1 *** Settings ***
   2 Documentation     Test suite for netconf device entity ownership handling during isolation.
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               Performs basic operations (Create, Read, Update and Delete or CRUD) on device
  12 ...               data mounted onto a netconf connector and verifies cluster recovery after
  13 ...               network isolation and rejoin of the Entity_Owner node.
  14 ...
  15 ...               The suite recognizes 3 nodes,
  16 ...               - "CONFIGURER" (the node that configures the device at the beginning and then
  17 ...               deconfigures it at the end).
  18 ...               - "SETTER" (the node that manipulates the data on the device)
  19 ...               - "CHECKER" (the node that checks the data on the device). The configured
  20 ...               device and the results of each data operation on it is expected to be visible
  21 ...               on all nodes. After each operation three test cases verify they can see the
  22 ...               result on their respective CONFIGURER,SETTER,CHECKER nodes.
  23 ...
  24 ...               The 3 nodes are configured by placing "node1", "node2" or "node3" into the
  25 ...               ${NODE_CONFIGURER}, ${NODE_SETTER} and ${NODE_CHECKER} variables. This makes
  26 ...               a node "CONFIGURER", "SETTER" and "CHECKER" respectively.
  27 ...               The "nodeX" name refers to the node with its IP address configured with the
  28 ...               ${ODL_SYSTEM_X_IP} variable where the "X" is 1, 2 or 3.
  29 ...
  30 ...               The suite verifies the presence of the device and the integrity
  31 ...               of device data for nodes that have at least one of the roles
  32 ...               ("CONFIGURER", "SETTER" and "CHECKER") assigned.
  33 ...
  34 ...               TODO: Multiple improvements are possible, but apply them to entity.robot at the same time please.
  35 Suite Setup       Setup_Everything
  36 Suite Teardown    Teardown_Everything
  37 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  38 Library           Collections
  39 Library           RequestsLibrary
  40 Library           OperatingSystem
  41 Library           String
  42 Library           SSHLibrary    timeout=10s
  43 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  44 Resource          ${CURDIR}/../../../libraries/FailFast.robot
  45 Resource          ${CURDIR}/../../../libraries/KarafKeywords.robot
  46 Resource          ${CURDIR}/../../../libraries/NetconfKeywords.robot
  47 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  48 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  49 Resource          ${CURDIR}/../../../libraries/Utils.robot
  50 Resource          ${CURDIR}/../../../variables/Variables.robot
  51
  52 *** Variables ***
  53 ${DEVICE_CHECK_TIMEOUT}    60s
  54 ${DEVICE_BOOT_TIMEOUT}    100s
  55 ${CLUSTER_RECOVERY_TIMEOUT}    300s
  56 ${DEVICE_NAME}    netconf-test-device
  57
  58 *** Test Cases ***
  59 Start_Testtool
  60     [Documentation]    Deploy and start test tool, then wait for all its devices to become online.
  61     NetconfKeywords.Install_And_Start_Testtool    device-count=1    schemas=${CURDIR}/../../../variables/netconf/CRUD/schemas
  62
  63 Check_Device_Is_Not_Mounted_At_Beginning
  64     [Documentation]    Sanity check making sure our device is not there. Fail if found.
  65     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=${node1_session}
  66     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=${node2_session}
  67     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=${node3_session}
  68
  69 Configure_Device_On_Netconf
  70     [Documentation]    Use node 1 to configure a testtool device on Netconf connector.
  71     NetconfKeywords.Configure_Device_In_Netconf    ${DEVICE_NAME}    device_type=configure-via-topology    session=${node1_session}
  72
  73 Wait_For_Device_To_Become_Visible_For_All_Nodes
  74     [Documentation]    Wait for the whole cluster to see the device.
  75     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=${node1_session}
  76     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=${node2_session}
  77     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=${node3_session}
  78
  79 Check_Config_Data_Before_Data_Creation
  80     [Documentation]    Check if there really is no data present on none of the nodes
  81     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node1_session}    ${empty_data}
  82     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node2_session}    ${empty_data}
  83     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node3_session}    ${empty_data}
  84
  85 Create_Device_Data
  86     [Documentation]    Create some data on the device and propagate it throughout the cluster.
  87     ${template_as_string}=    BuiltIn.Set_Variable    {'DEVICE_NAME': '${DEVICE_NAME}'}
  88     TemplatedRequests.Post_As_Xml_Templated    ${directory_with_template_folders}${/}dataorig    ${template_as_string}    session=${node2_session}
  89
  90 Check_Config_Data_After_Data_Creation
  91     [Documentation]    Check if the data we just added into the cluster is really there
  92     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node1_session}    ${original_data}
  93     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node2_session}    ${original_data}
  94     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node3_session}    ${original_data}
  95
  96 Find_And_Isolate_Device_Entity_Owner
  97     [Documentation]    Simulate a failure of the owner of the entity that represents the device.
  98     ${owner}    ${followers}=    ClusterManagement.Get_Owner_And_Successors_For_device    ${DEVICE NAME}    netconf    1
  99     Log    ${followers}
 100     Length Should Be    ${followers}    2    Wrong count of followers returned
 101     BuiltIn.Set_Suite_Variable    ${original_device_owner}    ${owner}
 102     BuiltIn.Set_Suite_Variable    ${follower1}    ${followers}[0]
 103     BuiltIn.Set_Suite_Variable    ${follower2}    ${followers}[1]
 104     ${original_device_owner_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${original_device_owner}
 105     ${follower1_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${follower1}
 106     ${follower2_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${follower2}
 107     BuiltIn.Set_Suite_Variable    ${original_device_owner_session}
 108     BuiltIn.Set_Suite_Variable    ${follower1_session}
 109     BuiltIn.Set_Suite_Variable    ${follower2_session}
 110     #ClusterManagement.Kill_Single_Member    ${owner}
 111     ClusterManagement.Isolate_Member_From_List_Or_All    ${owner}
 112
 113 Wait_For_New_Owner_To_Appear
 114     [Documentation]    Wait for the cluster to recover from the failure and choose a new owner for the entity.
 115     [Tags]    critical
 116     BuiltIn.Wait_Until_Keyword_Succeeds    ${CLUSTER_RECOVERY_TIMEOUT}    1s    Check_Owner_Reconfigured
 117
 118 Check_Config_Data_Before_Modification_With_Original_Owner_Down
 119     [Documentation]    Check if data is present and retrievable from follower nodes
 120     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${follower1_session}    ${original_data}
 121     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${follower2_session}    ${original_data}
 122
 123 Modify_Device_Data_When_Original_Owner_Is_Down
 124     [Documentation]    Attempt to modify the data on the device after recovery and see if it still works.
 125     ${template_as_string}=    BuiltIn.Set_Variable    {'DEVICE_NAME': '${DEVICE_NAME}'}
 126     TemplatedRequests.Put_As_Xml_Templated    ${directory_with_template_folders}${/}datamod1    ${template_as_string}    session=${follower1_session}
 127
 128 Check_Config_Data_After_Modification_With_Original_Owner_Down
 129     [Documentation]    Check if data is written correctly when original owner is shut down
 130     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${follower1_session}    ${modified_data}
 131     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${follower2_session}    ${modified_data}
 132
 133 Rejoin_Original_Entity_Owner
 134     [Documentation]    Restart the original entity owner and see if it can still see the device and the new data on it.
 135     ClusterManagement.Rejoin_Member_From_List_Or_All    ${original_device_owner}
 136
 137 Check_Config_Data_After_Original_Owner_Restart
 138     [Documentation]    Sanity check if we still can retrieve our data from the original device owner.
 139     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${original_device_owner_session}    ${modified_data}
 140     [Teardown]    Utils.Report_Failure_Due_To_Bug    8999
 141
 142 Modify_Device_Data_With_Original_Owner
 143     [Documentation]    Check that the original owner of the entity is still able to modify the data on the device
 144     ${template_as_string}=    BuiltIn.Set_Variable    {'DEVICE_NAME': '${DEVICE_NAME}'}
 145     TemplatedRequests.Put_As_Xml_Templated    ${directory_with_template_folders}${/}datamod2    ${template_as_string}    session=${original_device_owner_session}
 146
 147 Check_Config_Data_After_Modification_With_Original_Owner_Up
 148     [Documentation]    Check if data has really been written as we expect. Fails if Modify_Device_Data_With_Original_Owner fails.
 149     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node1_session}    ${modified_data_2}
 150     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node2_session}    ${modified_data_2}
 151     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    ${node3_session}    ${modified_data_2}
 152
 153 Deconfigure_Device_In_Netconf
 154     [Documentation]    Make request to deconfigure the device on Netconf connector and see if it works.
 155     [Tags]    critical
 156     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 157     NetconfKeywords.Remove_Device_From_Netconf    ${DEVICE_NAME}    session=${node1_session}
 158     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=${node1_session}
 159     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=${node2_session}
 160     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=${node3_session}
 161
 162 *** Keywords ***
 163 Setup_Everything
 164     [Documentation]    Setup everything needed for the test cases.
 165     # Setup resources used by the suite.
 166     SetupUtils.Setup_Utils_For_Setup_And_Teardown    http_timeout=2
 167     ClusterManagement.ClusterManagement_Setup
 168     NetconfKeywords.Setup_Netconf_Keywords    create_session_for_templated_requests=False
 169     ${node1_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=1
 170     ${node2_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=2
 171     ${node3_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=3
 172     BuiltIn.Set_Suite_Variable    ${node1_session}
 173     BuiltIn.Set_Suite_Variable    ${node2_session}
 174     BuiltIn.Set_Suite_Variable    ${node3_session}
 175     # Constants that are not meant to be overriden by the users
 176     BuiltIn.Set_Suite_Variable    ${directory_with_template_folders}    ${CURDIR}/../../../variables/netconf/CRUD
 177     BuiltIn.Set_Suite_Variable    ${empty_data}    <data xmlns="${ODL_NETCONF_NAMESPACE}"></data>
 178     ${cont}=    BuiltIn.Set_Variable    <data xmlns="${ODL_NETCONF_NAMESPACE}"><cont xmlns="urn:opendaylight:test:netconf:crud"><l>
 179     ${contend}=    BuiltIn.Set_Variable    </l></cont></data>
 180     BuiltIn.Set_Suite_Variable    ${original_data}    ${cont}Content${contend}
 181     BuiltIn.Set_Suite_Variable    ${modified_data}    ${cont}Modified Content${contend}
 182     BuiltIn.Set_Suite_Variable    ${modified_data_2}    ${cont}Another Modified Content${contend}
 183
 184 Teardown_Everything
 185     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 186     RequestsLibrary.Delete_All_Sessions
 187     NetconfKeywords.Stop_Testtool
 188
 189 Check_Config_Data
 190     [Arguments]    ${session}    ${expected}    ${contains}=False
 191     [Documentation]    Check that the data on the device matches the specified expectations.
 192     ...    TODO: Needs to be extracted into a suitable Resource as there is
 193     ...    the same code in at least two other suites (CRUD and clustered
 194     ...    CRUD).
 195     ${url}=    Builtin.Set_Variable    ${CONFIG_API}/network-topology:network-topology/topology/topology-netconf/node/${DEVICE_NAME}/yang-ext:mount
 196     ${data}=    TemplatedRequests.Get_As_Xml_From_Uri    ${url}    session=${session}
 197     BuiltIn.Run_Keyword_Unless    ${contains}    BuiltIn.Should_Be_Equal_As_Strings    ${data}    ${expected}
 198     BuiltIn.Run_Keyword_If    ${contains}    BuiltIn.Should_Contain    ${data}    ${expected}
 199
 200 Check_Owner_Reconfigured
 201     [Documentation]    Check whether the entity owner changed. Fail if not or no owner found.
 202     Log    Original Owner Index: ${original_device_owner}
 203     Log    Follower 1: node${follower1}
 204     Log    Follower 2: node${follower2}
 205     ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Candidates_For_Device    ${DEVICE_NAME}    netconf    ${follower1}
 206     BuiltIn.Should_Not_Be_Equal_As_Integers    ${owner}    ${original_device_owner}