csit/suites/netconf/clustering/entity.robot

   1 *** Settings ***
   2 Documentation     Test suite for netconf device entity ownership handling during outages.
   3 ...
   4 ...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               Perform basic operations (Create, Read, Update and Delete or CRUD) on device
  12 ...               data mounted onto a netconf connector and see if they work.
  13 ...
  14 ...               The suite recognizes 3 nodes, "CONFIGURER" (the node that configures the
  15 ...               device at the beginning and then deconfigures it at the end), "SETTER" (the
  16 ...               node that manipulates the data on the device) and "CHECKER" (the node that
  17 ...               checks the data on the device). The configured device and the results of each
  18 ...               data operation on it is expected to be visible on all nodes so after each
  19 ...               operation three test cases make sure they can see the result on their
  20 ...               respective nodes.
  21 ...
  22 ...               The 3 nodes are configured by placing "node1", "node2" or "node3" into the
  23 ...               ${NODE_CONFIGURER}, ${NODE_SETTER} and ${NODE_CHECKER} to make the node
  24 ...               a "CONFIGURER", "SETTER" and "CHECKER" respectively. The "nodeX" name refers
  25 ...               to the node with its IP address configured with the ${ODL_SYSTEM_X_IP}
  26 ...               variable where the "X" is 1, 2 or 3.
  27 ...
  28 ...               The suite checks the integrity of the presence of the device and the data
  29 ...               seen on the device only for nodes that have at least one of the roles
  30 ...               ("CONFIGURER", "SETTER" and "CHECKER") assigned. A better design would have
  31 ...               a "checker list" of sorts and have only one checking test case that runs
  32 ...               through the check list and performs the test on each node listed. However
  33 ...               this currently has fairly low priority due to Beryllium delivery date so
  34 ...               it was left out.
  35 Suite Setup       Setup_Everything
  36 Suite Teardown    Teardown_Everything
  37 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  38 Library           Collections
  39 Library           RequestsLibrary
  40 Library           OperatingSystem
  41 Library           String
  42 Library           SSHLibrary    timeout=10s
  43 Resource          ${CURDIR}/../../../libraries/ClusterKeywords.robot
  44 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  45 Resource          ${CURDIR}/../../../libraries/FailFast.robot
  46 Resource          ${CURDIR}/../../../libraries/KarafKeywords.robot
  47 Resource          ${CURDIR}/../../../libraries/NetconfKeywords.robot
  48 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  49 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  50 Resource          ${CURDIR}/../../../libraries/Utils.robot
  51 Variables         ${CURDIR}/../../../variables/Variables.py
  52
  53 *** Variables ***
  54 ${DEVICE_CHECK_TIMEOUT}    60s
  55 ${DEVICE_BOOT_TIMEOUT}    100s
  56 ${CLUSTER_RECOVERY_TIMEOUT}    120s
  57 ${DEVICE_NAME}    netconf-test-device
  58
  59 *** Test Cases ***
  60 Start_Testtool
  61     [Documentation]    Deploy and start test tool, then wait for all its devices to become online.
  62     NetconfKeywords.Install_And_Start_Testtool    device-count=1    schemas=${CURDIR}/../../../variables/netconf/CRUD/schemas
  63
  64 Check_Device_Is_Not_Mounted_At_Beginning
  65     [Documentation]    Sanity check making sure our device is not there. Fail if found.
  66     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node1
  67     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node2
  68     NetconfKeywords.Check_Device_Has_No_Netconf_Connector    ${DEVICE_NAME}    session=node3
  69
  70 Configure_Device_On_Netconf
  71     [Documentation]    Use node 1 to configure a testtool device on Netconf connector.
  72     NetconfKeywords.Configure_Device_In_Netconf    ${DEVICE_NAME}    device_type=configure-via-topology    session=node1
  73     [Teardown]    Utils.Report_Failure_Due_To_Bug    5089
  74
  75 Wait_For_Device_To_Become_Visible_For_All_Nodes
  76     [Documentation]    Wait for the whole cluster to see the device.
  77     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node1
  78     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node2
  79     NetconfKeywords.Wait_Device_Connected    ${DEVICE_NAME}    session=node3
  80
  81 Create_Device_Data
  82     [Documentation]    Create some data on the device and propagate it throughout the cluster.
  83     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${empty_data}
  84     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${empty_data}
  85     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${empty_data}
  86     ${template_as_string}=    BuiltIn.Set_Variable    {'DEVICE_NAME': '${DEVICE_NAME}'}
  87     TemplatedRequests.Post_As_Xml_Templated    ${directory_with_template_folders}${/}dataorig    ${template_as_string}    session=node2
  88     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${original_data}
  89     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${original_data}
  90     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${original_data}
  91
  92 Find_And_Shutdown_Device_Entity_Owner
  93     [Documentation]    Simulate a failure of the owner of the entity that represents the device.
  94     ${owner}    ${candidates}=    Get_Netconf_Entity_Info    ${DEVICE_NAME}    session=node1
  95     Length Should Be    ${candidates}    2    Wrong count of candidates returned
  96     BuiltIn.Set_Suite_Variable    ${original_device_owner}    ${owner}
  97     BuiltIn.Set_Suite_Variable    ${candidate1}    @{candidates}[0]
  98     BuiltIn.Set_Suite_Variable    ${candidate2}    @{candidates}[1]
  99     ClusterManagement.Kill_Single_Member    ${owner}
 100
 101 Wait_For_New_Owner_To_Appear
 102     [Documentation]    Wait for the cluster to recover from the failure and choose a new owner for the entity.
 103     [Tags]    critical
 104     BuiltIn.Wait_Until_Keyword_Succeeds    ${CLUSTER_RECOVERY_TIMEOUT}    1s    Check_Owner_Reconfigured    ${original_device_owner}
 105
 106 Modify_Device_Data_When_Original_Owner_Is_Down
 107     [Documentation]    Attempt to modify the data on the device after recovery and see if it still works.
 108     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node${candidate1}    ${original_data}
 109     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node${candidate2}    ${original_data}
 110     ${template_as_string}=    BuiltIn.Set_Variable    {'DEVICE_NAME': '${DEVICE_NAME}'}
 111     TemplatedRequests.Put_As_Xml_Templated    ${directory_with_template_folders}${/}datamod1    ${template_as_string}    session=node${candidate1}
 112     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node${candidate1}    ${modified_data}
 113     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node${candidate2}    ${modified_data}
 114     [Teardown]    Utils.Report_Failure_Due_To_Bug    4968
 115
 116 Restart_Original_Entity_Owner
 117     [Documentation]    Restart the original entity owner and see if it can still see the device and the new data on it.
 118     ClusterManagement.Start_Single_Member    ${original_device_owner}
 119     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node${original_device_owner}    ${modified_data}
 120     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 121
 122 Modify_Device_Data_With_Original_Owner
 123     [Documentation]    Check that the original owner of the entity is still able to modify the data on the device
 124     ${template_as_string}=    BuiltIn.Set_Variable    {'DEVICE_NAME': '${DEVICE_NAME}'}
 125     TemplatedRequests.Put_As_Xml_Templated    ${directory_with_template_folders}${/}datamod2    ${template_as_string}    session=node${original_device_owner}
 126     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node1    ${modified_data_2}
 127     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node2    ${modified_data_2}
 128     BuiltIn.Wait_Until_Keyword_Succeeds    ${DEVICE_CHECK_TIMEOUT}    1s    Check_Config_Data    node3    ${modified_data_2}
 129     [Teardown]    Utils.Report_Failure_Due_To_Bug    5761
 130
 131 Deconfigure_Device_In_Netconf
 132     [Documentation]    Make request to deconfigure the device on Netconf connector and see if it works.
 133     [Tags]    critical
 134     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 135     NetconfKeywords.Remove_Device_From_Netconf    ${DEVICE_NAME}    session=node1
 136     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node1
 137     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node2
 138     NetconfKeywords.Wait_Device_Fully_Removed    ${DEVICE_NAME}    session=node3
 139
 140 *** Keywords ***
 141 Setup_Everything
 142     [Documentation]    Setup everything needed for the test cases.
 143     # Setup resources used by the suite.
 144     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 145     ClusterManagement.ClusterManagement_Setup
 146     NetconfKeywords.Setup_Netconf_Keywords    create_session_for_templated_requests=False
 147     ClusterKeywords.Create_Controller_Sessions
 148     # TODO: Refactor the suite to use ClusterManagement.Resolve_Http_Session_For_Member instead of these 3 "hardcoded" sessions.
 149     RequestsLibrary.Create_Session    node1    http://${ODL_SYSTEM_1_IP}:${RESTCONFPORT}    headers=${HEADERS_XML}    auth=${AUTH}
 150     RequestsLibrary.Create_Session    node2    http://${ODL_SYSTEM_2_IP}:${RESTCONFPORT}    headers=${HEADERS_XML}    auth=${AUTH}
 151     RequestsLibrary.Create_Session    node3    http://${ODL_SYSTEM_3_IP}:${RESTCONFPORT}    headers=${HEADERS_XML}    auth=${AUTH}
 152     # Constants that are not meant to be overriden by the users
 153     BuiltIn.Set_Suite_Variable    ${directory_with_template_folders}    ${CURDIR}/../../../variables/netconf/CRUD
 154     BuiltIn.Set_Suite_Variable    ${empty_data}    <data xmlns="${ODL_NETCONF_NAMESPACE}"></data>
 155     ${cont}=    BuiltIn.Set_Variable    <data xmlns="${ODL_NETCONF_NAMESPACE}"><cont xmlns="urn:opendaylight:test:netconf:crud"><l>
 156     ${contend}=    BuiltIn.Set_Variable    </l></cont></data>
 157     BuiltIn.Set_Suite_Variable    ${original_data}    ${cont}Content${contend}
 158     BuiltIn.Set_Suite_Variable    ${modified_data}    ${cont}Modified Content${contend}
 159     BuiltIn.Set_Suite_Variable    ${modified_data_2}    ${cont}Another Modified Content${contend}
 160
 161 Teardown_Everything
 162     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 163     RequestsLibrary.Delete_All_Sessions
 164     NetconfKeywords.Stop_Testtool
 165
 166 Check_Config_Data
 167     [Arguments]    ${node}    ${expected}    ${contains}=False
 168     [Documentation]    Check that the data on the device matches the specified expectations.
 169     ...    TODO: Needs to be extracted into a suitable Resource as there is
 170     ...    the same code in at least two other suites (CRUD and clustered
 171     ...    CRUD).
 172     ${url}=    Builtin.Set_Variable    ${CONFIG_API}/network-topology:network-topology/topology/topology-netconf/node/${DEVICE_NAME}/yang-ext:mount
 173     ${data}=    TemplatedRequests.Get_As_Xml_From_Uri    ${url}    session=${node}
 174     BuiltIn.Run_Keyword_Unless    ${contains}    BuiltIn.Should_Be_Equal_As_Strings    ${data}    ${expected}
 175     BuiltIn.Run_Keyword_If    ${contains}    BuiltIn.Should_Contain    ${data}    ${expected}
 176
 177 Get_Netconf_Entity_Info
 178     [Arguments]    ${entity}    ${session}
 179     [Documentation]    Get owner and candidates for the specified netconf entity
 180     ...    TODO: Merge with ClusterKeywords.Get_Cluster_Entity_Owner which
 181     ...    contains most of the code from this keyword.
 182     ${entity_type}=    BuiltIn.Set_Variable    netconf-node/${entity}
 183     ${candidates_list}=    Create List
 184     ${data}=    Utils.Get Data From URI    ${session}    /restconf/operational/entity-owners:entity-owners
 185     Log    ${data}
 186     ${clear_data}=    Replace String    ${data}    /general-entity:entity[general-entity:name='    ${EMPTY}
 187     ${clear_data}=    Replace String    ${clear_data}    ']    ${EMPTY}
 188     ${json}=    RequestsLibrary.To Json    ${clear_data}
 189     ${entity_type_list}=    Get From Dictionary    &{json}[entity-owners]    entity-type
 190     ${entity_type_index}=    Get Index From List Of Dictionaries    ${entity_type_list}    type    ${entity_type}
 191     Should Not Be Equal    ${entity_type_index}    -1    No Entity Owner found for ${entity_type}
 192     ${entity_list}=    Get From Dictionary    @{entity_type_list}[${entity_type_index}]    entity
 193     ${entity_index}=    Utils.Get Index From List Of Dictionaries    ${entity_list}    id    ${entity}
 194     Should Not Be Equal    ${entity_index}    -1    Device ${entity} not found in Entity Owner ${entity_type}
 195     ${entity_owner}=    Get From Dictionary    @{entity_list}[${entity_index}]    owner
 196     Should Not Be Empty    ${entity_owner}    No owner found for ${entity}
 197     ${owner}=    Replace String    ${entity_owner}    member-    ${EMPTY}
 198     ${owner}=    Convert To Integer    ${owner}
 199     ${entity_candidates_list}=    Get From Dictionary    @{entity_list}[${entity_index}]    candidate
 200     ${list_length}=    Get Length    ${entity_candidates_list}
 201     : FOR    ${entity_candidate}    IN    @{entity_candidates_list}
 202     \    ${candidate}=    Replace String    &{entity_candidate}[name]    member-    ${EMPTY}
 203     \    ${candidate}=    Convert To Integer    ${candidate}
 204     \    Append To List    ${candidates_list}    ${candidate}
 205     Remove Values From List    ${candidates_list}    ${owner}
 206     [Return]    ${owner}    ${candidates_list}
 207
 208 Check_Owner_Reconfigured
 209     [Arguments]    ${original_device_owner}
 210     [Documentation]    Check whether the entity owner changed. Fail if not or no owner found.
 211     ${owner}    ${candidates}=    Get_Netconf_Entity_Info    ${DEVICE_NAME}    session=${candidate1}
 212     BuiltIn.Should_Not_Be_Equal_As_Integers    ${owner}    ${original_device_owner}