csit/suites/netconf/restperfclient/mdsal.robot

   1 *** Settings ***
   2 Documentation     netconf-restperfclient MDSAL performance test suite.
   3 ...
   4 ...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               Perform given count of update operations on ODL MDSAL. In first half the
  12 ...               requests are directed directly to MDSAL via Restconf and in the second
  13 ...               half the MDSAL is mounted onto a netconf connector and the reqursts are
  14 ...               directed to that connector. In both cases the netconf-testtool-restperfclient
  15 ...               tool is used to generate and send the requests and the requests are sent
  16 ...               synchronously as the netconf connector mounted MDSAL does not support
  17 ...               asynchronous requests. The restperfclient is used to generate the "update"
  18 ...               requests, the "create" request is issued in a sepate test case.
  19 Suite Setup       Setup_Everything
  20 Suite Teardown    Teardown_Everything
  21 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
  22 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
  23 Library           DateTime
  24 Library           RequestsLibrary
  25 Library           OperatingSystem
  26 Library           SSHLibrary    timeout=10s
  27 Resource          ${CURDIR}/../../../libraries/NetconfKeywords.robot
  28 Resource          ${CURDIR}/../../../libraries/NetconfViaRestconf.robot
  29 Resource          ${CURDIR}/../../../libraries/NexusKeywords.robot
  30 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  31 Resource          ${CURDIR}/../../../libraries/Utils.robot
  32 Variables         ${CURDIR}/../../../variables/Variables.py
  33
  34 *** Variables ***
  35 ${DIRECTORY_WITH_TEMPLATE_FOLDERS}    ${CURDIR}/../../../variables/netconf/RestPerfClient
  36 ${REQUEST_COUNT}    65536
  37
  38 *** Test Cases ***
  39 Create_Test_Data_For_Direct_Access
  40     [Documentation]    Send some sample test data into the device and check that the request went OK.
  41     NetconfViaRestconf.Post_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars    {}
  42
  43 Run_RestPerfClient_Directly_On_MDSAL
  44     [Documentation]    Deploy and execute restperfclient, asking it to send the specified amount of requests to the MDSAL via Restconf.
  45     [Timeout]    ${DIRECT_MDSAL_TIMEOUT_FOR_TESTCASE}
  46     ${restperfclientlog}=    Utils.Get_Log_File_Name    restperfclient    direct
  47     BuiltIn.Set_Suite_Variable    ${restperfclientlog}    ${restperfclientlog}
  48     SSHLibrary.Switch_Connection    ${restperfclient}
  49     SSHLibrary.Put_File    ${CURDIR}/../../../variables/netconf/RestPerfClient/request1.json
  50     ${filename}=    NexusKeywords.Deploy_Test_Tool    netconf    netconf-testtool    rest-perf-client
  51     SSHLibrary.Set_Client_Configuration    timeout=${DIRECT_MDSAL_TIMEOUT}
  52     ${options}=    BuiltIn.Set_Variable    --ip ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --edits ${REQUEST_COUNT}
  53     ${options}=    BuiltIn.Set_Variable    ${options} --edit-content request1.json --async-requests false
  54     ${options}=    BuiltIn.Set_Variable    ${options} --auth ${ODL_RESTCONF_USER} ${ODL_RESTCONF_PASSWORD}
  55     ${options}=    BuiltIn.Set_Variable    ${options} --destination
  56     ${prefix}=    NexusKeywords.Compose_Full_Java_Command    -Xmx1G -XX:MaxPermSize=256M -jar ${filename} ${options}
  57     BuiltIn.Set_Suite_Variable    ${command_prefix}    ${prefix}
  58     ${command}    BuiltIn.Set_Variable    ${command_prefix} /restconf/config/car:cars
  59     BuiltIn.Log    Running restperfclient: ${command}
  60     Set_Known_Bug_Id    5413
  61     Execute_Command_Passes    ${command} >${restperfclientlog} 2>&1
  62     Set_Unknown_Bug_Id
  63     ${result}=    SSHLibrary.Execute_Command    grep "FINISHED. Execution time:" ${restperfclientlog}
  64     BuiltIn.Should_Not_Be_Equal    '${result}'    ''
  65
  66 Check_For_Failed_Direct_MDSAL_Requests
  67     [Documentation]    Make sure there are no failed requests in the restperfclient log.
  68     ...    This is a separate test case to distinguish between restperfclient
  69     ...    failure and failed requests. Failed requests are rejected because
  70     ...    we don't want to test performance of ODL rejecting our requests.
  71     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  72     ${result}=    SSHLibrary.Execute_Command    grep "thread timed out" ${restperfclientlog}
  73     BuiltIn.Should_Be_Equal    '${result}'    ''
  74     ${result}=    SSHLibrary.Execute_Command    grep "Request failed" ${restperfclientlog}
  75     BuiltIn.Should_Be_Equal    '${result}'    ''
  76     ${result}=    SSHLibrary.Execute_Command    grep "Status code" ${restperfclientlog}
  77     BuiltIn.Should_Be_Equal    '${result}'    ''
  78
  79 Cleanup_And_Collect_For_Direct_Access
  80     [Documentation]    Cleanup the test data produced by the direct MDSAL access.
  81     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  82     NetconfViaRestconf.Delete_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars-delete    {}
  83     SSHLibrary.Get_File    ${restperfclientlog}
  84
  85 Create_Test_Data_For_Connector_Access
  86     [Documentation]    Create the test data container again so it is ready for the netconf connector test.
  87     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  88     NetconfViaRestconf.Post_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars    {}
  89
  90 Configure_ODL_As_A_Device_On_Netconf
  91     [Documentation]    Configure ODL MDSAL Northbound as a Netconf device on a Netconf connector.
  92     NetconfKeywords.Configure_Device_In_Netconf    odl-mdsal-northbound-via-netconf-connector    device_address=${ODL_SYSTEM_IP}    device_port=${ODL_NETCONF_MDSAL_PORT}    device_user=${ODL_NETCONF_USER}    device_password=${ODL_NETCONF_PASSWORD}
  93     NetconfKeywords.Wait_Device_Connected    odl-mdsal-northbound-via-netconf-connector
  94
  95 Run_RestPerfClient_Through_Netconf_Connector
  96     [Documentation]    Ask RestPerfClient to send the requests to the MDSAL mapped via a netconf connector.
  97     [Timeout]    ${NETCONF_CONNECTOR_MDSAL_TIMEOUT_FOR_TESTCASE}
  98     ${restperfclientlog}=    Utils.Get_Log_File_Name    restperfclient    netconf-connector
  99     BuiltIn.Set_Suite_Variable    ${restperfclientlog}    ${restperfclientlog}
 100     SSHLibrary.Switch_Connection    ${restperfclient}
 101     SSHLibrary.Set_Client_Configuration    timeout=${NETCONF_CONNECTOR_MDSAL_TIMEOUT}
 102     ${command}    BuiltIn.Set_Variable    ${command_prefix} /restconf/config/network-topology:network-topology/topology/topology-netconf/node/odl-mdsal-northbound-via-netconf-connector/yang-ext:mount/car:cars
 103     BuiltIn.Log    Running restperfclient: ${command}
 104     Set_Known_Bug_Id    5413
 105     Execute_Command_Passes    ${command} >${restperfclientlog} 2>&1
 106     Set_Unknown_Bug_Id
 107     ${result}=    SSHLibrary.Execute_Command    grep "FINISHED. Execution time:" ${restperfclientlog}
 108     BuiltIn.Should_Not_Be_Equal    '${result}'    ''
 109
 110 Check_For_Failed_Netconf_Connector_Requests
 111     [Documentation]    Make sure there are no failed requests in the restperfclient log.
 112     ...    This is a separate test case to distinguish between restperfclient
 113     ...    failure and failed requests. Failed requests are rejected because
 114     ...    we don't want to test performance of ODL rejecting our requests.
 115     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 116     Set_Known_Bug_Id    5581
 117     ${result}=    SSHLibrary.Execute_Command    grep "thread timed out" ${restperfclientlog}
 118     BuiltIn.Should_Be_Equal    '${result}'    ''
 119     Set_Unknown_Bug_Id
 120     ${result}=    SSHLibrary.Execute_Command    grep "Request failed" ${restperfclientlog}
 121     BuiltIn.Should_Be_Equal    '${result}'    ''
 122     ${result}=    SSHLibrary.Execute_Command    grep "Status code" ${restperfclientlog}
 123     BuiltIn.Should_Be_Equal    '${result}'    ''
 124
 125 Deconfigure_ODL_From_Netconf
 126     [Documentation]    Deconfigure the ODL MDSAL Northbound attached to a Netconf connector.
 127     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 128     NetconfKeywords.Remove_Device_From_Netconf    odl-mdsal-northbound-via-netconf-connector
 129
 130 Cleanup_And_Collect_For_Connector_Access
 131     [Documentation]    Delete the test data produced by the Netconf connector MDSAL access.
 132     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 133     NetconfViaRestconf.Delete_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars-delete    {}
 134     SSHLibrary.Get_File    ${restperfclientlog}
 135
 136 *** Keywords ***
 137 Setup_Everything
 138     [Documentation]    Setup everything needed for the test cases.
 139     # Setup resources used by the suite.
 140     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 141     NetconfKeywords.Setup_Netconf_Keywords
 142     # Connect to the tools system (rest-perf-client)
 143     ${restperfclient}=    SSHKeywords.Open_Connection_To_Tools_System
 144     BuiltIn.Set_Suite_Variable    ${restperfclient}    ${restperfclient}
 145     # Initialize artifact deployment infrastructure.
 146     ${testtool}=    SSHLibrary.Get Connection
 147     BuiltIn.Set_Suite_Variable    ${testtool}    ${testtool.index}
 148     # Calculate timeouts
 149     ${value}=    BuiltIn.Evaluate    ${REQUEST_COUNT}/50+10
 150     Utils.Set_User_Configurable_Variable_Default    DIRECT_MDSAL_TIMEOUT    ${value} s
 151     ${value}=    DateTime.Add_Time_To_Time    ${DIRECT_MDSAL_TIMEOUT}    60s    result_format=compact
 152     Utils.Set_User_Configurable_Variable_Default    DIRECT_MDSAL_TIMEOUT_FOR_TESTCASE    ${value}
 153     ${value}=    BuiltIn.Evaluate    ${REQUEST_COUNT}/10+10
 154     Utils.Set_User_Configurable_Variable_Default    NETCONF_CONNECTOR_MDSAL_TIMEOUT    ${value} s
 155     ${value}=    DateTime.Add_Time_To_Time    ${NETCONF_CONNECTOR_MDSAL_TIMEOUT}    60s    result_format=compact
 156     Utils.Set_User_Configurable_Variable_Default    NETCONF_CONNECTOR_MDSAL_TIMEOUT_FOR_TESTCASE    ${value}
 157
 158 Teardown_Everything
 159     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 160     Teardown_Netconf_Via_Restconf
 161     RequestsLibrary.Delete_All_Sessions
 162     SSHLibrary.Switch_Connection    ${testtool}
 163     SSHLibrary.Close_Connection