csit/suites/netconf/restperfclient/mdsal.robot

   1 *** Settings ***
   2 Documentation       netconf-restperfclient MDSAL performance test suite.
   3 ...
   4 ...                 Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...                 Perform given count of update operations on ODL MDSAL. In first half the
  12 ...                 requests are directed directly to MDSAL via Restconf and in the second
  13 ...                 half the MDSAL is mounted onto a netconf connector and the reqursts are
  14 ...                 directed to that connector. In both cases the netconf-testtool-restperfclient
  15 ...                 tool is used to generate and send the requests and the requests are sent
  16 ...                 synchronously as the netconf connector mounted MDSAL does not support
  17 ...                 asynchronous requests. The restperfclient is used to generate the "update"
  18 ...                 requests, the "create" request is issued in a sepate test case.
  19
  20 Library             RequestsLibrary
  21 Library             OperatingSystem
  22 Library             SSHLibrary    timeout=10s
  23 Resource            ${CURDIR}/../../../libraries/NetconfKeywords.robot
  24 Resource            ${CURDIR}/../../../libraries/NexusKeywords.robot
  25 Resource            ${CURDIR}/../../../libraries/RestPerfClient.robot
  26 Resource            ${CURDIR}/../../../libraries/SetupUtils.robot
  27 Resource            ${CURDIR}/../../../libraries/TemplatedRequests.robot
  28 Resource            ${CURDIR}/../../../libraries/Utils.robot
  29 Variables           ${CURDIR}/../../../variables/Variables.py
  30
  31 Suite Setup         Setup_Everything
  32 Suite Teardown      Teardown_Everything
  33 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
  34 Test Teardown       SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
  35
  36
  37 *** Variables ***
  38 ${DIRECTORY_WITH_TEMPLATE_FOLDERS}      ${CURDIR}/../../../variables/netconf/RestPerfClient
  39 ${REQUEST_COUNT}                        16384
  40 ${device_type}                          full-uri-device
  41 ${test_device}                          odl-mdsal-northbound-via-netconf-connector
  42
  43
  44 *** Test Cases ***
  45 Create_Test_Data_For_Direct_Access
  46     [Documentation]    Send some sample test data into the device and check that the request went OK.
  47     TemplatedRequests.Post_As_Xml_Templated    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars    {}
  48
  49 Run_RestPerfClient_Directly_On_MDSAL
  50     [Documentation]    Deploy and execute restperfclient, asking it to send the specified amount of requests to the MDSAL via Restconf.
  51     ${url}=    BuiltIn.Set_Variable    /rests/data/car:cars
  52     RestPerfClient.Invoke_Restperfclient    ${DIRECT_MDSAL_TIMEOUT}    ${url}    testcase=direct
  53
  54 Check_For_Failed_Direct_MDSAL_Requests
  55     [Documentation]    Make sure there are no failed requests in the restperfclient log.
  56     ...    This is a separate test case to distinguish between restperfclient
  57     ...    failure and failed requests. Failed requests are rejected because
  58     ...    we don't want to test performance of ODL rejecting our requests.
  59     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  60     ${result}=    RestPerfClient.Grep_Restperfclient_Log    thread timed out
  61     BuiltIn.Should_Be_Equal    '${result}'    ''
  62     ${result}=    RestPerfClient.Grep_Restperfclient_Log    Request failed
  63     BuiltIn.Should_Be_Equal    '${result}'    ''
  64     ${result}=    RestPerfClient.Grep_Restperfclient_Log    Status code
  65     BuiltIn.Should_Be_Equal    '${result}'    ''
  66
  67 Cleanup_And_Collect_For_Direct_Access
  68     [Documentation]    Cleanup the test data produced by the direct MDSAL access.
  69     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  70     RestPerfClient.Collect_From_Restperfclient
  71     TemplatedRequests.Delete_Templated    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars-delete    {}
  72
  73 Create_Test_Data_For_Connector_Access
  74     [Documentation]    Create the test data container again so it is ready for the netconf connector test.
  75     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  76     TemplatedRequests.Post_As_Xml_Templated    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars    {}
  77
  78 Configure_ODL_As_A_Device_On_Netconf
  79     [Documentation]    Configure ODL MDSAL Northbound as a Netconf device on a Netconf connector.
  80     NetconfKeywords.Configure_Device_In_Netconf
  81     ...    ${test_device}
  82     ...    device_type=${device_type}
  83     ...    device_address=${ODL_SYSTEM_IP}
  84     ...    device_port=${ODL_NETCONF_MDSAL_PORT}
  85     ...    device_user=${ODL_NETCONF_USER}
  86     ...    device_password=${ODL_NETCONF_PASSWORD}
  87     NetconfKeywords.Wait_Device_Connected    ${test_device}
  88
  89 Run_RestPerfClient_Through_Netconf_Connector
  90     [Documentation]    Ask RestPerfClient to send the requests to the MDSAL mapped via netconf topology device.
  91     ${url}=    BuiltIn.Set_Variable
  92     ...    /rests/data/network-topology:network-topology/topology\=topology-netconf/node\=${test_device}/yang-ext:mount/car:cars
  93     RestPerfClient.Invoke_Restperfclient    ${NETCONF_CONNECTOR_MDSAL_TIMEOUT}    ${url}    testcase=netconf-connector
  94
  95 Check_For_Failed_Netconf_Connector_Requests
  96     [Documentation]    Make sure there are no failed requests in the restperfclient log.
  97     ...    This is a separate test case to distinguish between restperfclient
  98     ...    failure and failed requests. Failed requests are rejected because
  99     ...    we don't want to test performance of ODL rejecting our requests.
 100     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 101     Set_Known_Bug_Id    5581
 102     ${result}=    RestPerfClient.Grep_Restperfclient_Log    thread timed out
 103     BuiltIn.Should_Be_Equal    '${result}'    ''
 104     Set_Unknown_Bug_Id
 105     ${result}=    RestPerfClient.Grep_Restperfclient_Log    Request failed
 106     BuiltIn.Should_Be_Equal    '${result}'    ''
 107     ${result}=    RestPerfClient.Grep_Restperfclient_Log    Status code
 108     BuiltIn.Should_Be_Equal    '${result}'    ''
 109
 110 Deconfigure_ODL_From_Netconf
 111     [Documentation]    Deconfigure the ODL MDSAL Northbound attached to a Netconf connector.
 112     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 113     NetconfKeywords.Remove_Device_From_Netconf    ${test_device}
 114
 115 Cleanup_And_Collect_For_Connector_Access
 116     [Documentation]    Delete the test data produced by the Netconf connector MDSAL access.
 117     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 118     RestPerfClient.Collect_From_Restperfclient
 119     TemplatedRequests.Delete_Templated    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars-delete    {}
 120
 121
 122 *** Keywords ***
 123 Setup_Everything
 124     [Documentation]    Setup everything needed for the test cases.
 125     # Setup resources used by the suite.
 126     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 127     NetconfKeywords.Setup_Netconf_Keywords
 128     RestPerfClient.Setup_Restperfclient
 129     ${device_type}=    BuiltIn.Set_Variable_If    ${USE_NETCONF_CONNECTOR}==${True}    default    ${device_type}
 130     BuiltIn.Set_Suite_Variable    ${device_type}
 131     # Calculate timeouts
 132     ${value}=    BuiltIn.Evaluate    ${REQUEST_COUNT}/50+10
 133     Utils.Set_User_Configurable_Variable_Default    DIRECT_MDSAL_TIMEOUT    ${value} s
 134     ${value}=    BuiltIn.Evaluate    ${REQUEST_COUNT}/10+20
 135     Utils.Set_User_Configurable_Variable_Default    NETCONF_CONNECTOR_MDSAL_TIMEOUT    ${value} s
 136
 137 Teardown_Everything
 138     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 139     RequestsLibrary.Delete_All_Sessions
 140     RestPerfClient.Teardown_Restperfclient