csit/suites/netconf/scale/max_devices.robot

   1 *** Settings ***
   2 Documentation     netconf-connector scaling test suite to find max connected devices
   3 ...
   4 ...               Copyright (c) 2019 Lumina Networks, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               Increasing numbers of netconf devices will be connected and cleaned up
  12 ...               while validating and profiling between each iteration.
  13 Suite Setup       Setup_Everything
  14 Suite Teardown    Teardown_Everything
  15 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  16 Library           Collections
  17 Library           String
  18 Library           SSHLibrary    timeout=10s
  19 Resource          ../../../libraries/CheckJVMResource.robot
  20 Resource          ../../../libraries/KarafKeywords.robot
  21 Resource          ../../../libraries/NetconfKeywords.robot
  22 Resource          ../../../libraries/SetupUtils.robot
  23 Resource          ../../../libraries/SSHKeywords.robot
  24 Resource          ../../../variables/Variables.robot
  25
  26 *** Variables ***
  27 ${INIT_DEVICE_COUNT}    250
  28 ${MAX_DEVICE_COUNT}    5000
  29 ${DEVICE_INCREMENT}    250
  30 ${DEVICE_NAME_BASE}    netconf-scaling-device
  31 ${DEVICE_TYPE}    full-uri-device
  32 ${BASE_PORT}      17830
  33 ${NUM_WORKERS}    10
  34 ${TIMEOUT_FACTOR}    2
  35 ${DEVICES_RESULT_FILE}    devices.csv
  36 ${INSTALL_TESTTOOL}    True
  37 ${TESTTOOL_EXECUTABLE}    ${EMPTY}
  38
  39 *** Test Cases ***
  40 Find Max Netconf Devices
  41     [Documentation]    Find max number of switches starting from ${MIN_SWITCHES} till reaching ${MAX_SWITCHES} in steps of ${STEP_SWITCHES}
  42     ${error_message} =    BuiltIn.Set Variable    Failure initializing suite
  43     ${maximum_devices} =    BuiltIn.Set Variable    ${0}
  44     ${discover_time} =    BuiltIn.Set Variable    0
  45     ${start} =    BuiltIn.Convert to Integer    ${INIT_DEVICE_COUNT}
  46     ${stop} =    BuiltIn.Convert to Integer    ${MAX_DEVICE_COUNT}
  47     ${increment} =    BuiltIn.Convert to Integer    ${DEVICE_INCREMENT}
  48     ${schema_dir} =    Run Keyword If    "${SCHEMA_MODEL}" == "juniper"    Get Juniper Device Schemas
  49     ...    ELSE    Set Variable    none
  50     Run Keyword And Ignore Error    CheckJVMResource.Get JVM Memory
  51     FOR    ${devices}    IN RANGE    ${start}    ${stop+1}    ${increment}
  52         ${timeout} =    BuiltIn.Evaluate    ${devices}*${TIMEOUT_FACTOR}
  53         Log To Console    Starting Iteration with ${devices} devices
  54         Run Keyword If    "${INSTALL_TESTTOOL}"=="True"    NetconfKeywords.Install_And_Start_Testtool    debug=false    schemas=${schema_dir}    device-count=${devices}
  55         ...    ELSE    NetconfKeywords.Start_Testtool    ${TESTTOOL_EXECUTABLE}    debug=false    schemas=${schema_dir}    device-count=${devices}
  56         ${status}    ${result} =    Run Keyword And Ignore Error    NetconfKeywords.Perform_Operation_On_Each_Device    Configure_Device    timeout=${timeout}
  57         Exit For Loop If    '${status}' == 'FAIL'
  58         ${status}    ${result} =    Run Keyword And Ignore Error    NetconfKeywords.Perform_Operation_On_Each_Device    Wait_Connected    timeout=${timeout}
  59         Exit For Loop If    '${status}' == 'FAIL'
  60         ${status}    ${result} =    Run Keyword And Ignore Error    Issue_Requests_On_Devices    ${TOOLS_SYSTEM_IP}    ${devices}
  61         ...    ${NUM_WORKERS}
  62         Exit For Loop If    '${status}' == 'FAIL'
  63         ${status}    ${result} =    Run Keyword And Ignore Error    NetconfKeywords.Perform_Operation_On_Each_Device    Wait_Connected    timeout=${timeout}
  64         Exit For Loop If    '${status}' == 'FAIL'
  65         ${status}    ${result} =    Run Keyword And Ignore Error    NetconfKeywords.Perform_Operation_On_Each_Device    Deconfigure_Device    timeout=${timeout}
  66         Exit For Loop If    '${status}' == 'FAIL'
  67         ${status}    ${result} =    Run Keyword And Ignore Error    NetconfKeywords.Perform_Operation_On_Each_Device    Check_Device_Deconfigured    timeout=${timeout}
  68         Exit For Loop If    '${status}' == 'FAIL'
  69         ${maximum_devices} =    Set Variable    ${devices}
  70         Run Keyword And Ignore Error    CheckJVMResource.Get JVM Memory
  71         NetconfKeywords.Stop_Testtool
  72     END
  73     [Teardown]    Run Keywords    NetconfKeywords.Stop_Testtool
  74     ...    AND    Collect_Data_Points    ${maximum_devices}
  75     ...    AND    Run Keyword And Ignore Error    CheckJVMResource.Get JVM Memory
  76
  77 *** Keywords ***
  78 Collect_Data_Points
  79     [Arguments]    ${devices}
  80     [Documentation]    Parse and Log relevant information when Scale test finishes
  81     OperatingSystem.Append To File    ${DEVICES_RESULT_FILE}    Max Devices\n
  82     OperatingSystem.Append To File    ${DEVICES_RESULT_FILE}    ${devices}\n
  83
  84 Issue_Requests_On_Devices
  85     [Arguments]    ${client_ip}    ${expected_count}    ${worker_count}
  86     [Documentation]    Spawn the specified count of worker threads to issue a GET request to each of the devices.
  87     ${current_ssh_connection}=    SSHLibrary.Get Connection
  88     SSHLibrary.Open_Connection    ${client_ip}
  89     SSHKeywords.Flexible_Mininet_Login
  90     SSHLibrary.Write    python getter.py --odladdress=${ODL_SYSTEM_IP} --count=${expected_count} --name=${device_name_base} --workers=${worker_count}
  91     FOR    ${number}    IN RANGE    1    ${expected_count}+1
  92         Read_Python_Tool_Operation_Result    ${number}
  93     END
  94     SSHLibrary.Read_Until_Prompt
  95     SSHLibrary.Close_Connection
  96     SSHKeywords.Restore Current SSH Connection From Index    ${current_ssh_connection.index}
  97
  98 Setup_Everything
  99     [Documentation]    Setup everything needed for the test cases.
 100     # Setup resources used by the suite.
 101     RequestsLibrary.Create_Session    operational    http://${ODL_SYSTEM_IP}:${RESTCONFPORT}${OPERATIONAL_API}    auth=${AUTH}
 102     SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
 103     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 104     NetconfKeywords.Setup_Netconf_Keywords
 105     # Deploy testing tools.
 106     SSHLibrary.Put_File    ${CURDIR}/../../../../tools/netconf_tools/getter.py
 107     SSHLibrary.Put_File    ${CURDIR}/../../../libraries/AuthStandalone.py
 108     ${device_type}=    BuiltIn.Set_Variable_If    ${USE_NETCONF_CONNECTOR}==${True}    default    ${device_type}
 109     BuiltIn.Set_Suite_Variable    ${device_type}
 110
 111 Teardown_Everything
 112     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 113     RequestsLibrary.Delete_All_Sessions
 114     NetconfKeywords.Stop_Testtool
 115
 116 Configure_Device
 117     [Arguments]    ${current_name}
 118     [Documentation]    Operation for configuring the device.
 119     KarafKeywords.Log_Message_To_Controller_Karaf    Configuring device ${current_name} to Netconf
 120     NetconfKeywords.Configure_Device_In_Netconf    ${current_name}    device_type=${device_type}    device_port=${current_port}
 121     KarafKeywords.Log_Message_To_Controller_Karaf    Device ${current_name} configured
 122
 123 Wait_Connected
 124     [Arguments]    ${current_name}
 125     [Documentation]    Operation for waiting until the device is connected.
 126     KarafKeywords.Log_Message_To_Controller_Karaf    Waiting for device ${current_name} to connect
 127     NetconfKeywords.Wait_Device_Connected    ${current_name}    period=0.5s    timeout=120s
 128     KarafKeywords.Log_Message_To_Controller_Karaf    Device ${current_name} connected
 129
 130 Read_Python_Tool_Operation_Result
 131     [Arguments]    ${number}
 132     [Documentation]    Read and process a report line emitted from the Python tool that corresponds to the device with the given number.
 133     ${test}=    SSHLibrary.Read_Until_Regexp    \\n
 134     ${test}=    String.Split_String    ${test}    |
 135     ${response}=    Collections.Get_From_List    ${test}    0
 136     ${message}=    Collections.Get_From_List    ${test}    1
 137     BuiltIn.Run_Keyword_If    '${response}' == 'ERROR'    Fail    Error getting data: ${message}
 138     ${start}=    Collections.Get_From_List    ${test}    1
 139     ${stop}=    Collections.Get_From_List    ${test}    2
 140     ${ellapsed}=    Collections.Get_From_List    ${test}    3
 141     BuiltIn.Log    DATA REQUEST RESULT: Device=${number} StartTime=${start} StopTime=${stop} EllapsedTime=${ellapsed}
 142     ${data}=    Collections.Get_From_List    ${test}    4
 143     ${expected}=    BuiltIn.Set_Variable    '<data xmlns="${ODL_NETCONF_NAMESPACE}"></data>'
 144     BuiltIn.Should_Be_Equal_As_Strings    ${data}    ${expected}
 145
 146 Deconfigure_Device
 147     [Arguments]    ${current_name}
 148     [Documentation]    Operation for deconfiguring the device.
 149     KarafKeywords.Log_Message_To_Controller_Karaf    Deconfiguring device ${current_name}
 150     NetconfKeywords.Remove_Device_From_Netconf    ${current_name}
 151     KarafKeywords.Log_Message_To_Controller_Karaf    Device ${current_name} deconfigured
 152
 153 Check_Device_Deconfigured
 154     [Arguments]    ${current_name}
 155     [Documentation]    Operation for making sure the device is really deconfigured.
 156     KarafKeywords.Log_Message_To_Controller_Karaf    Waiting for device ${current_name} to disappear
 157     NetconfKeywords.Wait_Device_Fully_Removed    ${current_name}    period=0.5s    timeout=120s
 158     KarafKeywords.Log_Message_To_Controller_Karaf    Device ${current_name} removed
 159
 160 Get Juniper Device Schemas
 161     OperatingSystem.Run    git clone https://github.com/Juniper/yang.git
 162     OperatingSystem.Run    mkdir /tmp/junos_19.4R1
 163     OperatingSystem.Run    find yang/19.4/19.4R1/junos -type f -name '*yang' -exec cp {} /tmp/junos_19.4R1/ \\;
 164     OperatingSystem.Run    cp yang/19.4/19.4R1/common/* /tmp/junos_19.4R1/
 165     OperatingSystem.List Directory    /tmp/junos_19.4R1/
 166     [Return]    /tmp/junos_19.4R1