csit/suites/netconf/clustering/outages.robot

   1 *** Settings ***
   2 Documentation     netconf cluster node outage test suite (CRUD operations).
   3 ...
   4 ...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               Perform one of the basic operations (Create, Read, Update and Delete or CRUD)
  12 ...               on device data mounted onto a netconf connector while one of the nodes is
  13 ...               down and see if they work. Then bring the dead node up and check that it sees
  14 ...               the operations that were made while it was down are visible on it as well.
  15 ...
  16 ...               The node is brought down before each of the "Create", "Update" and "Delete"
  17 ...               operations and brought and back up after these operations. Before the dead
  18 ...               node is brought up, a test case makes sure the operation is properly
  19 ...               propagated within the cluster.
  20 ...
  21 ...               Currently each of the 3 operations is done once. "Create" is done while
  22 ...               node 1 is down, "Update" while node 2 is down and "Delete" while node 3
  23 ...               is down.
  24 Suite Setup       Setup_Everything
  25 Suite Teardown    Teardown_Everything
  26 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  27 Library           Collections
  28 Library           RequestsLibrary
  29 Library           OperatingSystem
  30 Library           String
  31 Library           SSHLibrary    timeout=10s
  32 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  33 Resource          ${CURDIR}/../../../libraries/FailFast.robot
  34 Resource          ${CURDIR}/../../../libraries/KarafKeywords.robot
  35 Resource          ${CURDIR}/../../../libraries/NetconfKeywords.robot
  36 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  37 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  38 Resource          ${CURDIR}/../../../libraries/Utils.robot
  39 Variables         ${CURDIR}/../../../variables/Variables.py
  40
  41 *** Variables ***
  42 ${DEVICE_CHECK_TIMEOUT}    60s
  43 ${DEVICE_BOOT_TIMEOUT}    100s
  44 ${DEVICE_NAME}    netconf-test-device
  45
  46 *** Test Cases ***
  47 Start_Testtool
  48     [Documentation]    Deploy and start test tool, then wait for all its devices to become online.
  49     NetconfKeywords.Install_And_Start_Testtool    device-count=1    schemas=${CURDIR}/../../../variables/netconf/CRUD/schemas
  50
  51 Check_Device_Is_Not_Mounted_At_Beginning
  52     [Documentation]    Sanity check making sure our device is not there. Fail if found.
  53     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node1
  54     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node2
  55     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node3
  56
  57 Configure_Device_On_Netconf
  58     [Documentation]    Use node 1 to configure a testtool device on Netconf connector
  59     NetconfKeywords.Configure_Device_In_Netconf    ${DEVICE_NAME}    device_type=configure-via-topology    session=node1
  60     [Teardown]    Utils.Report_Failure_Due_To_Bug    5089
  61
  62 Wait_For_Device_To_Become_Visible_For_All_Nodes
  63     [Documentation]    Check that the cluster communication about a new Netconf device configuration works
  64     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node1
  65     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node2
  66     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node3
  67
  68 Check_Device_Data_Is_Seen_As_Empty_On_All_Nodes
  69     [Documentation]    Sanity check against possible data left-overs from previous suites. Also causes the suite to wait until the entire cluster sees the device and its data mount.
  70     ${config_topology}    ${operational_topology}=    Get_Topology    session=node1
  71     ${config_topology}    ${operational_topology}=    Get_Topology    session=node2
  72     ${config_topology}    ${operational_topology}=    Get_Topology    session=node3
  73     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${empty_data}
  74     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${empty_data}
  75     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${empty_data}
  76
  77 Kill_node1_Before_Create
  78     [Documentation]    Simulate node 1 crashes just before device data is created, fail if node 1 survives.
  79     ClusterManagement.Kill_Single_Member    1
  80
  81 Create_Device_Data_With_node1_Down
  82     [Documentation]    Check that the create requests work when node 1 is down.
  83     [Tags]    critical
  84     TemplatedRequests.Post_As_Xml_Templated    ${directory_with_template_folders}${/}dataorig    {'DEVICE_NAME': '${DEVICE_NAME}'}    session=node2
  85
  86 Check_New_Device_Data_Is_Visible_On_Nodes_Without_node1
  87     [Documentation]    Check that the new device data is propagated in the cluster even when node 1 is down.
  88     [Tags]    critical
  89     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${original_data}
  90     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${original_data}
  91
  92 Restart_node1_After_Create_And_Dump_Its_Topology_Data
  93     [Documentation]    Simulate node 1 restarted by admin just after device data is created and the change propagated in the cluster, fail if node 1 fails to boot.
  94     ClusterManagement.Start_Single_Member    1
  95     ${config_topology}    ${operational_topology}=    Get_Topology    session=node1
  96
  97 Check_New_Device_Data_Is_Visible_On_node1
  98     [Documentation]    Check that the created device data is propagated to node 1 as well.
  99     [Tags]    critical
 100     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_BOOT_TIMEOUT}    1s    Check_Config_Data    node1    ${original_data}
 101     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 102
 103 Kill_node2_Before_Modify
 104     [Documentation]    Simulate node 2 crashes just before device data is modified, fail if node 2 survives.
 105     ClusterManagement.Kill_Single_Member    2
 106
 107 Modify_Device_Data_With_node2_Down
 108     [Documentation]    Check that the modification requests work when node 2 is down.
 109     [Tags]    critical
 110     TemplatedRequests.Put_As_Xml_Templated    ${directory_with_template_folders}${/}datamod1    {'DEVICE_NAME': '${DEVICE_NAME}'}    session=node3
 111     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 112
 113 Check_Modified_Device_Data_Is_Visible_On_Nodes_Without_node2
 114     [Documentation]    Check that the device data modification is propagated in the cluster even when node 2 is down.
 115     [Tags]    critical
 116     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${modified_data}
 117     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${modified_data}
 118     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 119
 120 Restart_node2_After_Modify_And_Dump_Its_Topology_Data
 121     [Documentation]    Simulate node 2 restarted by admin just after device data is modified and the change propagated in the cluster, fail if node 2 fails to boot.
 122     ClusterManagement.Start_Single_Member    2
 123     ${config_topology}    ${operational_topology}=    Get_Topology    session=node2
 124
 125 Check_Modified_Device_Data_Is_Visible_On_node2
 126     [Documentation]    Check that the device data modification is propagated to node 2 as well.
 127     [Tags]    critical
 128     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_BOOT_TIMEOUT}    1s    Check_Config_Data    node2    ${modified_data}
 129     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 130
 131 Kill_node3_Before_Delete
 132     [Documentation]    Simulate node 3 crashes just before device data is deleted, fail if node 3 survives.
 133     ClusterManagement.Kill_Single_Member    3
 134
 135 Delete_Device_Data_With_node3_Down
 136     [Documentation]    Check that the data removal requests work when node 3 is down.
 137     [Tags]    critical
 138     TemplatedRequests.Delete_Templated    ${directory_with_template_folders}${/}datamod1    {'DEVICE_NAME': '${DEVICE_NAME}'}    session=node1
 139     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 140
 141 Check_Device_Data_Removal_Is_Visible_On_Nodes_Without_node3
 142     [Documentation]    Check that the device data removal is propagated in the cluster even when node 3 is down.
 143     [Tags]    critical
 144     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${empty_data}
 145     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${empty_data}
 146     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 147
 148 Restart_node3_After_Delete_And_Dump_Its_Topology_Data
 149     [Documentation]    Simulate node 3 restarted by admin just after device data is deleted and the change propagated in the cluster, fail if node 3 fails to boot.
 150     ClusterManagement.Start_Single_Member    3
 151     ${config_topology}    ${operational_topology}=    Get_Topology    session=node3
 152
 153 Check_Device_Data_Removal_Is_Visible_On_node3
 154     [Documentation]    Check that the device data removal is propagated to node 3 as well.
 155     [Tags]    critical
 156     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_BOOT_TIMEOUT}    1s    Check_Config_Data    node3    ${empty_data}
 157     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 158
 159 Deconfigure_Device_In_Netconf
 160     [Documentation]    Make request to deconfigure the device on Netconf connector to clean things up and also check that it still works after all the node outages.
 161     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 162     NetconfKeywords.Remove_Device_From_Netconf    ${DEVICE_NAME}    session=node1
 163
 164 Check_Device_Deconfigured
 165     [Documentation]    Check that the device deconfiguration is propagated throughout the cluster correctly.
 166     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node1
 167     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node2
 168     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node3
 169
 170 *** Keywords ***
 171 Setup_Everything
 172     [Documentation]    Setup everything needed for the test cases.
 173     # Setup resources used by the suite.
 174     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 175     ClusterManagement.ClusterManagement_Setup
 176     NetconfKeywords.Setup_Netconf_Keywords    create_session_for_templated_requests=False
 177     # TODO: Refactor the suite to use ClusterManagement.Resolve_Http_Session_For_Member instead of these 3 "hardcoded" sessions.
 178     RequestsLibrary.Create_Session    node1    http://${ODL_SYSTEM_1_IP}:${RESTCONFPORT}    headers=${HEADERS_XML}    auth=${AUTH}
 179     RequestsLibrary.Create_Session    node2    http://${ODL_SYSTEM_2_IP}:${RESTCONFPORT}    headers=${HEADERS_XML}    auth=${AUTH}
 180     RequestsLibrary.Create_Session    node3    http://${ODL_SYSTEM_3_IP}:${RESTCONFPORT}    headers=${HEADERS_XML}    auth=${AUTH}
 181     BuiltIn.Set_Suite_Variable    ${directory_with_template_folders}    ${CURDIR}/../../../variables/netconf/CRUD
 182     BuiltIn.Set_Suite_Variable    ${empty_data}    <data xmlns="${ODL_NETCONF_NAMESPACE}"></data>
 183     BuiltIn.Set_Suite_Variable    ${original_data}    <data xmlns="${ODL_NETCONF_NAMESPACE}"><cont xmlns="urn:opendaylight:test:netconf:crud"><l>Content</l></cont></data>
 184     BuiltIn.Set_Suite_Variable    ${modified_data}    <data xmlns="${ODL_NETCONF_NAMESPACE}"><cont xmlns="urn:opendaylight:test:netconf:crud"><l>Modified Content</l></cont></data>
 185     ${url}=    Builtin.Set_Variable    /network-topology:network-topology/topology/topology-netconf
 186     BuiltIn.Set_Suite_Variable    ${config_topology_url}    ${CONFIG_API}${url}
 187     BuiltIn.Set_Suite_Variable    ${operational_topology_url}    ${OPERATIONAL_API}${url}
 188
 189 Get_Topology_Core
 190     [Arguments]    ${session}
 191     [Documentation]    Get both versions of topology (config and operational), log them and return them for further processing.
 192     ${config_topology}=    TemplatedRequests.Get_As_Json_From_Uri    ${config_topology_url}    session=${session}
 193     BuiltIn.Log    ${config_topology}
 194     ${operational_topology}=    TemplatedRequests.Get_As_Json_From_Uri    ${operational_topology_url}    session=${session}
 195     BuiltIn.Log    ${operational_topology}
 196     [Return]    ${config_topology}    ${operational_topology}
 197
 198 Get_Topology
 199     [Arguments]    ${session}
 200     [Documentation]    Repeatedly try to get the topologies using Get_Topology_Core until either the request succeeds or boot timeout period expires.
 201     ${result}=    BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_BOOT_TIMEOUT}    1s    Get_Topology_Core    ${session}
 202     [Return]    ${result}
 203
 204 Teardown_Everything
 205     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 206     RequestsLibrary.Delete_All_Sessions
 207     NetconfKeywords.Stop_Testtool
 208
 209 Check_Device_Instance_Count
 210     [Arguments]    ${expected}    ${session}
 211     [Documentation]    Check that the specified session sees the specified count of instances of the test tool device.
 212     ${count}    NetconfKeywords.Count_Netconf_Connectors_For_Device    ${DEVICE_NAME}    session=${session}
 213     Builtin.Should_Be_Equal_As_Strings    ${count}    ${expected}
 214
 215 Check_Config_Data
 216     [Arguments]    ${node}    ${expected}    ${contains}=False
 217     [Documentation]    Check that the specified session sees the specified data in the test tool device.
 218     ${url}=    Builtin.Set_Variable    ${CONFIG_API}/network-topology:network-topology/topology/topology-netconf/node/${DEVICE_NAME}/yang-ext:mount
 219     ${data}=    TemplatedRequests.Get_As_Xml_From_Uri    ${url}    session=${node}
 220     BuiltIn.Run_Keyword_Unless    ${contains}    BuiltIn.Should_Be_Equal_As_Strings    ${data}    ${expected}
 221     BuiltIn.Run_Keyword_If    ${contains}    BuiltIn.Should_Contain    ${data}    ${expected}