Performance test suite for MDSAL northbound

This suite uses RestPerfClient on MDSAL northbound, via
Restconf. Similar to "performance.robot" but no device is
involved and thus no testtool deployment is necessary,
the only needed tool is restperfclient.

Change-Id: I117e790cd5434933d105ccbc29c6bcf7a9fca69e
Signed-off-by: Jozef Behran <jbehran@cisco.com>

diff --git a/csit/libraries/Utils.robot b/csit/libraries/Utils.robot
index f4c1bdf9ed69cf67fd051525a7c9974791abf4b4..9acdba054c7fa4c17f034a111d81192df7e7e1c8 100644 (file)
--- a/csit/libraries/Utils.robot
+++ b/csit/libraries/Utils.robot
@@ -413,13 +413,14 @@ Post Log Check
     [Return]    ${resp}
 
 Get Log File Name
-    [Arguments]    ${testtool}
+    [Arguments]    ${testtool}    ${testcase}=${EMPTY}
     [Documentation]    Get the name of the suite sanitized to be usable as a part of filename.
     ...    These names are used to constructs names of the log files produced
     ...    by the testing tools so two suites using a tool wont overwrite the
     ...    log files if they happen to run in one job.
     ${name}=    BuiltIn.Evaluate    """${SUITE_NAME}""".replace(" ","-").replace("/","-").replace(".","-")
-    [Return]    ${testtool}--${name}.log
+    ${suffix}=    BuiltIn.Set_Variable_If    '${testcase}' != ''    --${testcase}    ${EMPTY}
+    [Return]    ${testtool}--${name}${suffix}.log
 
 Set_User_Configurable_Variable_Default
     [Arguments]    ${name}    ${value}

diff --git a/csit/suites/netconf/restperfclient/mdsal.robot b/csit/suites/netconf/restperfclient/mdsal.robot
new file mode 100644 (file)
index 0000000..d4c4c11
--- /dev/null
+++ b/csit/suites/netconf/restperfclient/mdsal.robot
@@ -0,0 +1,157 @@
+*** Settings ***
+Documentation     netconf-restperfclient MDSAL performance test suite.
+...
+...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
+...
+...               This program and the accompanying materials are made available under the
+...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
+...               and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+...
+...               Perform given count of update operations on ODL MDSAL. In first half the
+...               requests are directed directly to MDSAL via Restconf and in the second
+...               half the MDSAL is mounted onto a netconf connector and the reqursts are
+...               directed to that connector. In both cases the netconf-testtool-restperfclient
+...               tool is used to generate and send the requests and the requests are sent
+...               synchronously as the netconf connector mounted MDSAL does not support
+...               asynchronous requests. The restperfclient is used to generate the "update"
+...               requests, the "create" request is issued in a sepate test case.
+Suite Setup       Setup_Everything
+Suite Teardown    Teardown_Everything
+Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
+Library           DateTime
+Library           RequestsLibrary
+Library           OperatingSystem
+Library           SSHLibrary    timeout=10s
+Resource          ${CURDIR}/../../../libraries/NetconfKeywords.robot
+Resource          ${CURDIR}/../../../libraries/NetconfViaRestconf.robot
+Resource          ${CURDIR}/../../../libraries/NexusKeywords.robot
+Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
+Resource          ${CURDIR}/../../../libraries/Utils.robot
+Variables         ${CURDIR}/../../../variables/Variables.py
+
+*** Variables ***
+${DIRECTORY_WITH_TEMPLATE_FOLDERS}    ${CURDIR}/../../../variables/netconf/RestPerfClient
+${REQUEST_COUNT}    65536
+
+*** Test Cases ***
+Create_Test_Data_For_Direct_Access
+    [Documentation]    Send some sample test data into the device and check that the request went OK.
+    NetconfViaRestconf.Post_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars    {}
+
+Run_RestPerfClient_Directly_On_MDSAL
+    [Documentation]    Deploy and execute restperfclient, asking it to send the specified amount of requests to the MDSAL via Restconf.
+    [Timeout]    ${DIRECT_MDSAL_TIMEOUT_FOR_TESTCASE}
+    ${restperfclientlog}=    Utils.Get_Log_File_Name    restperfclient    direct
+    BuiltIn.Set_Suite_Variable    ${restperfclientlog}    ${restperfclientlog}
+    SSHLibrary.Switch_Connection    ${restperfclient}
+    SSHLibrary.Put_File    ${CURDIR}/../../../variables/netconf/RestPerfClient/request1.json
+    ${filename}=    NexusKeywords.Deploy_Test_Tool    netconf    netconf-testtool    rest-perf-client
+    SSHLibrary.Set_Client_Configuration    timeout=${DIRECT_MDSAL_TIMEOUT}
+    ${options}=    BuiltIn.Set_Variable    --ip ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --edits ${REQUEST_COUNT}
+    ${options}=    BuiltIn.Set_Variable    ${options} --edit-content request1.json --async-requests false
+    ${options}=    BuiltIn.Set_Variable    ${options} --auth ${ODL_RESTCONF_USER} ${ODL_RESTCONF_PASSWORD}
+    ${options}=    BuiltIn.Set_Variable    ${options} --destination
+    ${prefix}=    NexusKeywords.Compose_Full_Java_Command    -Xmx1G -XX:MaxPermSize=256M -jar ${filename} ${options}
+    BuiltIn.Set_Suite_Variable    ${command_prefix}    ${prefix}
+    ${command}    BuiltIn.Set_Variable    ${command_prefix} /restconf/config/car:cars
+    BuiltIn.Log    Running restperfclient: ${command}
+    Set_Known_Bug_Id    5413
+    Execute_Command_Passes    ${command} >${restperfclientlog} 2>&1
+    Set_Unknown_Bug_Id
+    ${result}=    SSHLibrary.Execute_Command    grep "FINISHED. Execution time:" ${restperfclientlog}
+    BuiltIn.Should_Not_Be_Equal    '${result}'    ''
+
+Check_For_Failed_Direct_MDSAL_Requests
+    [Documentation]    Make sure there are no failed requests in the restperfclient log.
+    ...    This is a separate test case to distinguish between restperfclient
+    ...    failure and failed requests. Failed requests are rejected because
+    ...    we don't want to test performance of ODL rejecting our requests.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    ${result}=    SSHLibrary.Execute_Command    grep "Request failed" ${restperfclientlog}
+    BuiltIn.Should_Be_Equal    '${result}'    ''
+    ${result}=    SSHLibrary.Execute_Command    grep "Status code" ${restperfclientlog}
+    BuiltIn.Should_Be_Equal    '${result}'    ''
+
+Cleanup_And_Collect_For_Direct_Access
+    [Documentation]    Cleanup the test data produced by the direct MDSAL access.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    NetconfViaRestconf.Delete_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars-delete    {}
+    SSHLibrary.Get_File    ${restperfclientlog}
+
+Create_Test_Data_For_Connector_Access
+    [Documentation]    Create the test data container again so it is ready for the netconf connector test.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    NetconfViaRestconf.Post_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars    {}
+
+Configure_ODL_As_A_Device_On_Netconf
+    [Documentation]    Configure ODL MDSAL Northbound as a Netconf device on a Netconf connector.
+    NetconfKeywords.Configure_Device_In_Netconf    odl-mdsal-northbound-via-netconf-connector    device_address=${ODL_SYSTEM_IP}    device_port=${ODL_NETCONF_MDSAL_PORT}    device_user=${ODL_NETCONF_USER}    device_password=${ODL_NETCONF_PASSWORD}
+    NetconfKeywords.Wait_Device_Connected    odl-mdsal-northbound-via-netconf-connector
+
+Run_RestPerfClient_Through_Netconf_Connector
+    [Documentation]    Ask RestPerfClient to send the requests to the MDSAL mapped via a netconf connector.
+    [Timeout]    ${NETCONF_CONNECTOR_MDSAL_TIMEOUT_FOR_TESTCASE}
+    ${restperfclientlog}=    Utils.Get_Log_File_Name    restperfclient    netconf-connector
+    BuiltIn.Set_Suite_Variable    ${restperfclientlog}    ${restperfclientlog}
+    SSHLibrary.Switch_Connection    ${restperfclient}
+    SSHLibrary.Set_Client_Configuration    timeout=${NETCONF_CONNECTOR_MDSAL_TIMEOUT}
+    ${command}    BuiltIn.Set_Variable    ${command_prefix} /restconf/config/network-topology:network-topology/topology/topology-netconf/node/odl-mdsal-northbound-via-netconf-connector/yang-ext:mount/car:cars
+    BuiltIn.Log    Running restperfclient: ${command}
+    Set_Known_Bug_Id    5413
+    Execute_Command_Passes    ${command} >${restperfclientlog} 2>&1
+    Set_Unknown_Bug_Id
+    ${result}=    SSHLibrary.Execute_Command    grep "FINISHED. Execution time:" ${restperfclientlog}
+    BuiltIn.Should_Not_Be_Equal    '${result}'    ''
+
+Check_For_Failed_Netconf_Connector_Requests
+    [Documentation]    Make sure there are no failed requests in the restperfclient log.
+    ...    This is a separate test case to distinguish between restperfclient
+    ...    failure and failed requests. Failed requests are rejected because
+    ...    we don't want to test performance of ODL rejecting our requests.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    ${result}=    SSHLibrary.Execute_Command    grep "Request failed" ${restperfclientlog}
+    BuiltIn.Should_Be_Equal    '${result}'    ''
+    ${result}=    SSHLibrary.Execute_Command    grep "Status code" ${restperfclientlog}
+    BuiltIn.Should_Be_Equal    '${result}'    ''
+
+Deconfigure_ODL_From_Netconf
+    [Documentation]    Deconfigure the ODL MDSAL Northbound attached to a Netconf connector.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    NetconfKeywords.Remove_Device_From_Netconf    odl-mdsal-northbound-via-netconf-connector
+
+Cleanup_And_Collect_For_Connector_Access
+    [Documentation]    Delete the test data produced by the Netconf connector MDSAL access.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    NetconfViaRestconf.Delete_Xml_Template_Folder_Via_Restconf    ${DIRECTORY_WITH_TEMPLATE_FOLDERS}${/}cars-delete    {}
+    SSHLibrary.Get_File    ${restperfclientlog}
+
+*** Keywords ***
+Setup_Everything
+    [Documentation]    Setup everything needed for the test cases.
+    # Setup resources used by the suite.
+    SetupUtils.Setup_Utils_For_Setup_And_Teardown
+    NetconfKeywords.Setup_Netconf_Keywords
+    # Connect to the tools system (rest-perf-client)
+    ${restperfclient}=    SSHKeywords.Open_Connection_To_Tools_System
+    BuiltIn.Set_Suite_Variable    ${restperfclient}    ${restperfclient}
+    # Initialize artifact deployment infrastructure.
+    ${testtool}=    SSHLibrary.Get Connection
+    BuiltIn.Set_Suite_Variable    ${testtool}    ${testtool.index}
+    # Calculate timeouts
+    ${value}=    BuiltIn.Evaluate    ${REQUEST_COUNT}/50+10
+    Utils.Set_User_Configurable_Variable_Default    DIRECT_MDSAL_TIMEOUT    ${value} s
+    ${value}=    DateTime.Add_Time_To_Time    ${DIRECT_MDSAL_TIMEOUT}    60s    result_format=compact
+    Utils.Set_User_Configurable_Variable_Default    DIRECT_MDSAL_TIMEOUT_FOR_TESTCASE    ${value}
+    ${value}=    BuiltIn.Evaluate    ${REQUEST_COUNT}/10+10
+    Utils.Set_User_Configurable_Variable_Default    NETCONF_CONNECTOR_MDSAL_TIMEOUT    ${value} s
+    ${value}=    DateTime.Add_Time_To_Time    ${NETCONF_CONNECTOR_MDSAL_TIMEOUT}    60s    result_format=compact
+    Utils.Set_User_Configurable_Variable_Default    NETCONF_CONNECTOR_MDSAL_TIMEOUT_FOR_TESTCASE    ${value}
+
+Teardown_Everything
+    [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
+    Teardown_Netconf_Via_Restconf
+    RequestsLibrary.Delete_All_Sessions
+    SSHLibrary.Switch_Connection    ${testtool}
+    SSHLibrary.Close_Connection

diff --git a/csit/testplans/netconf-scale.txt b/csit/testplans/netconf-scale.txt
index 522e07002e4eb6204d9cca148a2f466d7ba0f7bd..8af40d584320c002f30d8acba2284b62bf71d35e 100644 (file)
--- a/csit/testplans/netconf-scale.txt
+++ b/csit/testplans/netconf-scale.txt
@@ -7,5 +7,6 @@
 # Place the suites in run order:
 integration/test/csit/suites/netconf/ready
 integration/test/csit/suites/netconf/restperfclient/performance.robot
+integration/test/csit/suites/netconf/restperfclient/mdsal.robot
 integration/test/csit/suites/netconf/scale/getmulti.robot
 integration/test/csit/suites/netconf/scale/getsingle.robot

diff --git a/csit/variables/netconf/RestPerfClient/cars-delete/config.uri b/csit/variables/netconf/RestPerfClient/cars-delete/config.uri
new file mode 100644 (file)
index 0000000..70fb5e1
--- /dev/null
+++ b/csit/variables/netconf/RestPerfClient/cars-delete/config.uri
@@ -0,0 +1 @@
+car:cars

diff --git a/csit/variables/netconf/RestPerfClient/cars/config.uri b/csit/variables/netconf/RestPerfClient/cars/config.uri
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/csit/variables/netconf/RestPerfClient/cars/data.xml b/csit/variables/netconf/RestPerfClient/cars/data.xml
new file mode 100644 (file)
index 0000000..dc91199
--- /dev/null
+++ b/csit/variables/netconf/RestPerfClient/cars/data.xml
@@ -0,0 +1,5 @@
+<cars xmlns="urn:opendaylight:params:xml:ns:yang:controller:config:sal-clustering-it:car">
+  <car-entry>
+    <id>KEEP</id>
+  </car-entry>
+</cars>