csit/suites/netconf/clustering/outages.robot

   1 *** Settings ***
   2 Documentation       netconf cluster node outage test suite (CRUD operations).
   3 ...
   4 ...                 Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...                 Perform one of the basic operations (Create, Read, Update and Delete or CRUD)
  12 ...                 on device data mounted onto a netconf connector while one of the nodes is
  13 ...                 down and see if they work. Then bring the dead node up and check that it sees
  14 ...                 the operations that were made while it was down are visible on it as well.
  15 ...
  16 ...                 The node is brought down before each of the "Create", "Update" and "Delete"
  17 ...                 operations and brought and back up after these operations. Before the dead
  18 ...                 node is brought up, a test case makes sure the operation is properly
  19 ...                 propagated within the cluster.
  20 ...
  21 ...                 Currently each of the 3 operations is done once. "Create" is done while
  22 ...                 node 1 is down, "Update" while node 2 is down and "Delete" while node 3
  23 ...                 is down.
  24
  25 Library             Collections
  26 Library             RequestsLibrary
  27 Library             OperatingSystem
  28 Library             String
  29 Library             SSHLibrary    timeout=10s
  30 Resource            ${CURDIR}/../../../libraries/ClusterManagement.robot
  31 Resource            ${CURDIR}/../../../libraries/FailFast.robot
  32 Resource            ${CURDIR}/../../../libraries/KarafKeywords.robot
  33 Resource            ${CURDIR}/../../../libraries/NetconfKeywords.robot
  34 Resource            ${CURDIR}/../../../libraries/SetupUtils.robot
  35 Resource            ${CURDIR}/../../../libraries/TemplatedRequests.robot
  36 Resource            ${CURDIR}/../../../libraries/Utils.robot
  37 Variables           ${CURDIR}/../../../variables/Variables.py
  38
  39 Suite Setup         Setup_Everything
  40 Suite Teardown      Teardown_Everything
  41 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  42
  43
  44 *** Variables ***
  45 ${DEVICE_CHECK_TIMEOUT}         60s
  46 ${DEVICE_BOOT_TIMEOUT}          100s
  47 ${DEVICE_NAME}                  netconf-test-device
  48 ${MASTER_CONNECT_TIMEOUT}       15s
  49
  50
  51 *** Test Cases ***
  52 Start_Testtool
  53     [Documentation]    Deploy and start test tool, then wait for all its devices to become online.
  54     NetconfKeywords.Install_And_Start_Testtool
  55     ...    device-count=1
  56     ...    schemas=${CURDIR}/../../../variables/netconf/CRUD/schemas
  57
  58 Check_Device_Is_Not_Mounted_At_Beginning
  59     [Documentation]    Sanity check making sure our device is not there. Fail if found.
  60     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node1
  61     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node2
  62     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node3
  63
  64 Configure_Device_On_Netconf
  65     [Documentation]    Use node 1 to configure a testtool device on Netconf connector
  66     NetconfKeywords.Configure_Device_In_Netconf
  67     ...    ${DEVICE_NAME}
  68     ...    device_type=configure-via-topology
  69     ...    session=node1
  70     [Teardown]    Utils.Report_Failure_Due_To_Bug    5089
  71
  72 Wait_For_Device_To_Become_Visible_For_All_Nodes
  73     [Documentation]    Check that the cluster communication about a new Netconf device configuration works
  74     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node1
  75     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node2
  76     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node3
  77
  78 Check_Device_Data_Is_Seen_As_Empty_On_All_Nodes
  79     [Documentation]    Sanity check against possible data left-overs from previous suites. Also causes the suite to wait until the entire cluster sees the device and its data mount.
  80     ${config_topology}    ${operational_topology}=    Get_Topology    session=node1
  81     ${config_topology}    ${operational_topology}=    Get_Topology    session=node2
  82     ${config_topology}    ${operational_topology}=    Get_Topology    session=node3
  83     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${empty_data}
  84     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${empty_data}
  85     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${empty_data}
  86
  87 Kill_node1_Before_Create
  88     [Documentation]    Simulate node 1 crashes just before device data is created, fail if node 1 survives.
  89     ClusterManagement.Kill_Single_Member    1
  90
  91 Create_Device_Data_With_node1_Down
  92     [Documentation]    Check that the create requests work when node 1 is down.
  93     ...    As ODL may be in the process of connecting possible new master to the device,
  94     ...    the action is retried few times.
  95     ...    TODO: Check exact status before retry. Carbon reports 404 instead of the correct 503.
  96     [Tags]    critical
  97     ${template_as_string}=    BuiltIn.Create_Dictionary    DEVICE_NAME=${DEVICE_NAME}
  98     BuiltIn.Wait_Until_Keyword_Succeeds
  99     ...    ${MASTER_CONNECT_TIMEOUT}
 100     ...    1s
 101     ...    TemplatedRequests.Post_As_Xml_Templated
 102     ...    ${directory_with_template_folders}${/}dataorig
 103     ...    ${template_as_string}
 104     ...    session=node2
 105
 106 Check_New_Device_Data_Is_Visible_On_Nodes_Without_node1
 107     [Documentation]    Check that the new device data is propagated in the cluster even when node 1 is down.
 108     [Tags]    critical
 109     BuiltIn.Wait_Until_Keyword_Succeeds
 110     ...    ${DEVICE_CHECK_TIMEOUT}
 111     ...    1s
 112     ...    Check_Config_Data
 113     ...    node2
 114     ...    ${original_data}
 115     BuiltIn.Wait_Until_Keyword_Succeeds
 116     ...    ${DEVICE_CHECK_TIMEOUT}
 117     ...    1s
 118     ...    Check_Config_Data
 119     ...    node3
 120     ...    ${original_data}
 121
 122 Restart_node1_After_Create_And_Dump_Its_Topology_Data
 123     [Documentation]    Simulate node 1 restarted by admin just after device data is created and the change propagated in the cluster, fail if node 1 fails to boot.
 124     ClusterManagement.Start_Single_Member    1
 125     ${config_topology}    ${operational_topology}=    Get_Topology    session=node1
 126
 127 Check_New_Device_Data_Is_Visible_On_node1
 128     [Documentation]    Check that the created device data is propagated to node 1 as well.
 129     [Tags]    critical
 130     BuiltIn.Wait_Until_Keyword_Succeeds
 131     ...    ${DEVICE_BOOT_TIMEOUT}
 132     ...    1s
 133     ...    Check_Config_Data
 134     ...    node1
 135     ...    ${original_data}
 136     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 137
 138 Kill_node2_Before_Modify
 139     [Documentation]    Simulate node 2 crashes just before device data is modified, fail if node 2 survives.
 140     ClusterManagement.Kill_Single_Member    2
 141
 142 Modify_Device_Data_With_node2_Down
 143     [Documentation]    Check that the modification requests work when node 2 is down.
 144     ...    As ODL may be in the process of connecting possible new master to the device,
 145     ...    the action is retried few times.
 146     ...    TODO: Check exact status before retry. Carbon reports 404 instead of the correct 503.
 147     [Tags]    critical
 148     BuiltIn.Wait_Until_Keyword_Succeeds
 149     ...    ${MASTER_CONNECT_TIMEOUT}
 150     ...    1s
 151     ...    TemplatedRequests.Put_As_Xml_Templated
 152     ...    ${directory_with_template_folders}${/}datamod1
 153     ...    {'DEVICE_NAME': '${DEVICE_NAME}'}
 154     ...    session=node3
 155     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 156
 157 Check_Modified_Device_Data_Is_Visible_On_Nodes_Without_node2
 158     [Documentation]    Check that the device data modification is propagated in the cluster even when node 2 is down.
 159     [Tags]    critical
 160     BuiltIn.Wait_Until_Keyword_Succeeds
 161     ...    ${DEVICE_CHECK_TIMEOUT}
 162     ...    1s
 163     ...    Check_Config_Data
 164     ...    node1
 165     ...    ${modified_data}
 166     BuiltIn.Wait_Until_Keyword_Succeeds
 167     ...    ${DEVICE_CHECK_TIMEOUT}
 168     ...    1s
 169     ...    Check_Config_Data
 170     ...    node3
 171     ...    ${modified_data}
 172     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 173
 174 Restart_node2_After_Modify_And_Dump_Its_Topology_Data
 175     [Documentation]    Simulate node 2 restarted by admin just after device data is modified and the change propagated in the cluster, fail if node 2 fails to boot.
 176     ClusterManagement.Start_Single_Member    2
 177     ${config_topology}    ${operational_topology}=    Get_Topology    session=node2
 178
 179 Check_Modified_Device_Data_Is_Visible_On_node2
 180     [Documentation]    Check that the device data modification is propagated to node 2 as well.
 181     [Tags]    critical
 182     BuiltIn.Wait_Until_Keyword_Succeeds
 183     ...    ${DEVICE_BOOT_TIMEOUT}
 184     ...    1s
 185     ...    Check_Config_Data
 186     ...    node2
 187     ...    ${modified_data}
 188     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 189
 190 Kill_node3_Before_Delete
 191     [Documentation]    Simulate node 3 crashes just before device data is deleted, fail if node 3 survives.
 192     ClusterManagement.Kill_Single_Member    3
 193
 194 Delete_Device_Data_With_node3_Down
 195     [Documentation]    Check that the data removal requests work when node 3 is down.
 196     ...    As ODL may be in the process of connecting possible new master to the device,
 197     ...    the action is retried few times.
 198     ...    TODO: Check exact status before retry. Carbon reports 404 instead of the correct 503.
 199     [Tags]    critical
 200     BuiltIn.Wait_Until_Keyword_Succeeds
 201     ...    ${MASTER_CONNECT_TIMEOUT}
 202     ...    1s
 203     ...    TemplatedRequests.Delete_Templated
 204     ...    ${directory_with_template_folders}${/}datamod1
 205     ...    {'DEVICE_NAME': '${DEVICE_NAME}'}
 206     ...    session=node1
 207     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 208
 209 Check_Device_Data_Removal_Is_Visible_On_Nodes_Without_node3
 210     [Documentation]    Check that the device data removal is propagated in the cluster even when node 3 is down.
 211     [Tags]    critical
 212     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${empty_data}
 213     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${empty_data}
 214     [Teardown]    Utils.Report_Failure_Due_To_Bug    5762
 215
 216 Restart_node3_After_Delete_And_Dump_Its_Topology_Data
 217     [Documentation]    Simulate node 3 restarted by admin just after device data is deleted and the change propagated in the cluster, fail if node 3 fails to boot.
 218     ClusterManagement.Start_Single_Member    3
 219     ${config_topology}    ${operational_topology}=    Get_Topology    session=node3
 220
 221 Check_Device_Data_Removal_Is_Visible_On_node3
 222     [Documentation]    Check that the device data removal is propagated to node 3 as well.
 223     [Tags]    critical
 224     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_BOOT_TIMEOUT}    1s    Check_Config_Data    node3    ${empty_data}
 225     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 226
 227 Deconfigure_Device_In_Netconf
 228     [Documentation]    Make request to deconfigure the device on Netconf connector to clean things up and also check that it still works after all the node outages.
 229     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 230     NetconfKeywords.Remove_Device_From_Netconf    ${DEVICE_NAME}    session=node1
 231
 232 Check_Device_Deconfigured
 233     [Documentation]    Check that the device deconfiguration is propagated throughout the cluster correctly.
 234     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node1
 235     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node2
 236     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node3
 237
 238
 239 *** Keywords ***
 240 Setup_Everything
 241     [Documentation]    Setup everything needed for the test cases.
 242     # Setup resources used by the suite.
 243     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 244     ClusterManagement.ClusterManagement_Setup
 245     NetconfKeywords.Setup_Netconf_Keywords    create_session_for_templated_requests=False
 246     # TODO: Refactor the suite to use ClusterManagement.Resolve_Http_Session_For_Member instead of these 3 "hardcoded" sessions.
 247     RequestsLibrary.Create_Session
 248     ...    node1
 249     ...    http://${ODL_SYSTEM_1_IP}:${RESTCONFPORT}
 250     ...    headers=${HEADERS_XML}
 251     ...    auth=${AUTH}
 252     RequestsLibrary.Create_Session
 253     ...    node2
 254     ...    http://${ODL_SYSTEM_2_IP}:${RESTCONFPORT}
 255     ...    headers=${HEADERS_XML}
 256     ...    auth=${AUTH}
 257     RequestsLibrary.Create_Session
 258     ...    node3
 259     ...    http://${ODL_SYSTEM_3_IP}:${RESTCONFPORT}
 260     ...    headers=${HEADERS_XML}
 261     ...    auth=${AUTH}
 262     BuiltIn.Set_Suite_Variable    ${directory_with_template_folders}    ${CURDIR}/../../../variables/netconf/CRUD
 263     BuiltIn.Set_Suite_Variable    ${empty_data}    <data xmlns="${ODL_NETCONF_NAMESPACE}"></data>
 264     BuiltIn.Set_Suite_Variable
 265     ...    ${original_data}
 266     ...    <data xmlns="${ODL_NETCONF_NAMESPACE}"><cont xmlns="urn:opendaylight:test:netconf:crud"><l>Content</l></cont></data>
 267     BuiltIn.Set_Suite_Variable
 268     ...    ${modified_data}
 269     ...    <data xmlns="${ODL_NETCONF_NAMESPACE}"><cont xmlns="urn:opendaylight:test:netconf:crud"><l>Modified Content</l></cont></data>
 270     ${url}=    Builtin.Set_Variable    /network-topology:network-topology/topology=topology-netconf
 271     BuiltIn.Set_Suite_Variable    ${config_topology_url}    ${REST_API}${url}
 272     BuiltIn.Set_Suite_Variable    ${operational_topology_url}    ${REST_API}${url}
 273
 274 Get_Topology_Core
 275     [Documentation]    Get both versions of topology (config and operational), log them and return them for further processing.
 276     [Arguments]    ${session}
 277     ${config_topology}=    TemplatedRequests.Get_As_Json_From_Uri    ${config_topology_url}    session=${session}
 278     BuiltIn.Log    ${config_topology}
 279     ${operational_topology}=    TemplatedRequests.Get_As_Json_From_Uri
 280     ...    ${operational_topology_url}
 281     ...    session=${session}
 282     BuiltIn.Log    ${operational_topology}
 283     RETURN    ${config_topology}    ${operational_topology}
 284
 285 Get_Topology
 286     [Documentation]    Repeatedly try to get the topologies using Get_Topology_Core until either the request succeeds or boot timeout period expires.
 287     [Arguments]    ${session}
 288     ${result}=    BuiltIn.Wait_Until_Keyword_Succeeds
 289     ...    ${DEVICE_BOOT_TIMEOUT}
 290     ...    1s
 291     ...    Get_Topology_Core
 292     ...    ${session}
 293     RETURN    ${result}
 294
 295 Teardown_Everything
 296     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 297     RequestsLibrary.Delete_All_Sessions
 298     NetconfKeywords.Stop_Testtool
 299
 300 Check_Device_Instance_Count
 301     [Documentation]    Check that the specified session sees the specified count of instances of the test tool device.
 302     [Arguments]    ${expected}    ${session}
 303     ${count}=    NetconfKeywords.Count_Netconf_Connectors_For_Device    ${DEVICE_NAME}    session=${session}
 304     Builtin.Should_Be_Equal_As_Strings    ${count}    ${expected}
 305
 306 Check_Config_Data
 307     [Documentation]    Check that the specified session sees the specified data in the test tool device.
 308     [Arguments]    ${node}    ${expected}    ${contains}=False
 309     ${url}=    Builtin.Set_Variable
 310     ...    ${REST_API}/network-topology:network-topology/topology=topology-netconf/node=${DEVICE_NAME}/yang-ext:mount?content=config
 311     ${data}=    TemplatedRequests.Get_As_Xml_From_Uri    ${url}    session=${node}
 312     IF    not ${contains}
 313         BuiltIn.Should_Be_Equal_As_Strings    ${data}    ${expected}
 314     END
 315     IF    ${contains}    BuiltIn.Should_Contain    ${data}    ${expected}