csit/suites/netconf/scale/max_devices.robot

   1 *** Settings ***
   2 Documentation       netconf-connector scaling test suite to find max connected devices
   3 ...
   4 ...                 Copyright (c) 2019 Lumina Networks, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...                 Increasing numbers of netconf devices will be connected and cleaned up
  12 ...                 while validating and profiling between each iteration.
  13
  14 Library             Collections
  15 Library             String
  16 Library             SSHLibrary    timeout=1000s
  17 Library             ../../../libraries/TopologyNetconfNodes.py
  18 Resource            ../../../libraries/KarafKeywords.robot
  19 Resource            ../../../libraries/NetconfKeywords.robot
  20 Resource            ../../../libraries/SetupUtils.robot
  21 Resource            ../../../libraries/SSHKeywords.robot
  22 Resource            ../../../variables/Variables.robot
  23
  24 Suite Setup         Setup_Everything
  25 Suite Teardown      Teardown_Everything
  26 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  27
  28
  29 *** Variables ***
  30 ${INIT_DEVICE_COUNT}        250
  31 ${MAX_DEVICE_COUNT}         100000
  32 ${DEVICE_INCREMENT}         5000
  33 ${DEVICE_NAME_BASE}         netconf-scaling-device
  34 ${DEVICE_TYPE}              full-uri-device
  35 ${BASE_PORT}                17830
  36 ${NUM_WORKERS}              500
  37 ${TIMEOUT_FACTOR}           3
  38 ${MIN_CONNECT_TIMEOUT}      300
  39 ${DEVICES_RESULT_FILE}      devices.csv
  40 ${INSTALL_TESTTOOL}         True
  41 ${TESTTOOL_EXECUTABLE}      ${EMPTY}
  42
  43
  44 *** Test Cases ***
  45 Find Max Netconf Devices
  46     [Documentation]    Find max number of switches starting from ${MIN_SWITCHES} till reaching ${MAX_SWITCHES} in steps of ${STEP_SWITCHES}
  47     ${error_message} =    BuiltIn.Set Variable    Failure initializing suite
  48     ${maximum_devices} =    BuiltIn.Set Variable    ${0}
  49     ${discover_time} =    BuiltIn.Set Variable    0
  50     ${start} =    BuiltIn.Convert to Integer    ${INIT_DEVICE_COUNT}
  51     ${stop} =    BuiltIn.Convert to Integer    ${MAX_DEVICE_COUNT}
  52     ${increment} =    BuiltIn.Convert to Integer    ${DEVICE_INCREMENT}
  53     IF    "${SCHEMA_MODEL}" == "juniper"
  54         ${schema_dir} =    Get Juniper Device Schemas
  55     ELSE
  56         ${schema_dir} =    Set Variable    none
  57     END
  58     ${INSTALL_TESTTOOL} =    Set Variable If    '${IS_KARAF_APPL}' == 'False'    False    True
  59     ${TESTTOOL_EXECUTABLE} =    Set Variable If    '${IS_KARAF_APPL}' == 'False'    ${NETCONF_FILENAME}    ${EMPTY}
  60     ${SCHEMAS} =    Set Variable If
  61     ...    '${IS_KARAF_APPL}' == 'False'
  62     ...    ${CURDIR}/../../../variables/netconf/CRUD/schemas
  63     ...    ${schema_dir}
  64     ${restconf_url} =    BuiltIn.Set_Variable    http://${ODL_SYSTEM_IP}:${RESTCONFPORT}/rests
  65     ${device_names} =    BuiltIn.Set_Variable    []
  66     FOR    ${devices}    IN RANGE    ${start}    ${stop+1}    ${increment}
  67         ${timeout} =    BuiltIn.Evaluate    ${devices}*${TIMEOUT_FACTOR}
  68         ${timeout} =    Set Variable If    ${timeout} > ${MIN_CONNECT_TIMEOUT}    ${timeout}    ${MIN_CONNECT_TIMEOUT}
  69         Log To Console    Starting Iteration with ${devices} devices
  70         IF    "${INSTALL_TESTTOOL}"=="True"
  71             NetconfKeywords.Install_And_Start_Testtool
  72             ...    debug=false
  73             ...    schemas=${schema_dir}
  74             ...    device-count=${devices}
  75             ...    log_response=False
  76         ELSE
  77             NetconfKeywords.Start_Testtool
  78             ...    ${TESTTOOL_EXECUTABLE}
  79             ...    debug=false
  80             ...    schemas=${SCHEMAS}
  81             ...    device-count=${devices}
  82             ...    log_response=False
  83         END
  84         ${devices_to_configure} =    BuiltIn.Evaluate    ${devices} - len(${device_names})
  85         ${first_id} =    BuiltIn.Evaluate    len(${device_names}) + 1
  86         ${device_names} =    TopologyNetconfNodes.Configure Device Range
  87         ...    restconf_url=${restconf_url}
  88         ...    device_name_prefix=${DEVICE_NAME_BASE}
  89         ...    device_ipaddress=${TOOLS_SYSTEM_IP}
  90         ...    device_port=17830
  91         ...    device_count=${devices_to_configure}
  92         ...    first_device_id=${first_id}
  93         TopologyNetconfNodes.Await Devices Connected
  94         ...    restconf_url=${restconf_url}
  95         ...    device_names=${device_names}
  96         ...    deadline_seconds=${timeout}
  97         ${status}    ${result} =    Run Keyword And Ignore Error
  98         ...    Issue_Requests_On_Devices
  99         ...    ${TOOLS_SYSTEM_IP}
 100         ...    ${devices}
 101         ...    ${NUM_WORKERS}
 102         IF    '${status}' == 'FAIL'            BREAK
 103         ${maximum_devices} =    Set Variable    ${devices}
 104         NetconfKeywords.Stop_Testtool
 105     END
 106     [Teardown]    Run Keywords    NetconfKeywords.Stop_Testtool
 107     ...    AND    Collect_Data_Points    ${maximum_devices}
 108
 109
 110 *** Keywords ***
 111 Collect_Data_Points
 112     [Documentation]    Parse and Log relevant information when Scale test finishes
 113     [Arguments]    ${devices}
 114     OperatingSystem.Append To File    ${DEVICES_RESULT_FILE}    Max Devices\n
 115     OperatingSystem.Append To File    ${DEVICES_RESULT_FILE}    ${devices}\n
 116
 117 Issue_Requests_On_Devices
 118     [Documentation]    Spawn the specified count of worker threads to issue a GET request to each of the devices.
 119     [Arguments]    ${client_ip}    ${expected_count}    ${worker_count}
 120     # FIXME: this keyword is nearly duplicated in the getmulti.robot suite. need to move it to a common lib
 121     ${current_ssh_connection} =    SSHLibrary.Get Connection
 122     SSHLibrary.Open_Connection    ${client_ip}
 123     SSHKeywords.Flexible_Mininet_Login
 124     SSHLibrary.Write
 125     ...    python getter.py --odladdress=${ODL_SYSTEM_IP} --count=${expected_count} --name=${device_name_base} --workers=${worker_count}
 126     SSHLibrary.Read_Until    ${DEFAULT_LINUX_PROMPT_STRICT}
 127     SSHLibrary.Write
 128     ...    python getter.py --odladdress=${ODL_SYSTEM_IP} --count=${expected_count} --name=${device_name_base} --workers=${worker_count}
 129     FOR    ${number}    IN RANGE    1    ${expected_count}+1
 130         Read_Python_Tool_Operation_Result    ${number}
 131     END
 132     SSHLibrary.Read_Until    ${DEFAULT_LINUX_PROMPT_STRICT}
 133     SSHLibrary.Close_Connection
 134     SSHKeywords.Restore Current SSH Connection From Index    ${current_ssh_connection.index}
 135
 136 Setup_Everything
 137     [Documentation]    Setup everything needed for the test cases.
 138     # Setup resources used by the suite.
 139     RequestsLibrary.Create_Session    operational    http://${ODL_SYSTEM_IP}:${RESTCONFPORT}${REST_API}    auth=${AUTH}
 140     SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
 141     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 142     NetconfKeywords.Setup_Netconf_Keywords
 143     # Deploy testing tools.
 144     SSHLibrary.Put_File    ${CURDIR}/../../../../tools/netconf_tools/getter.py
 145     SSHLibrary.Put_File    ${CURDIR}/../../../libraries/AuthStandalone.py
 146     ${device_type} =    BuiltIn.Set_Variable_If    ${USE_NETCONF_CONNECTOR}==${True}    default    ${device_type}
 147     BuiltIn.Set_Suite_Variable    ${device_type}
 148
 149 Teardown_Everything
 150     [Documentation]    Teardown the test infrastructure, perform cleanup and release all resources.
 151     RequestsLibrary.Delete_All_Sessions
 152     NetconfKeywords.Stop_Testtool
 153
 154 Read_Python_Tool_Operation_Result
 155     [Documentation]    Read and process a report line emitted from the Python tool that corresponds to the device with the given number.
 156     [Arguments]    ${number}
 157     ${test} =    SSHLibrary.Read_Until_Regexp    \\n
 158     ${test} =    String.Split_String    ${test}    |
 159     ${response} =    Collections.Get_From_List    ${test}    0
 160     ${message} =    Collections.Get_From_List    ${test}    1
 161     IF    '${response}' == 'ERROR'    Fail    Error getting data: ${message}
 162     ${start} =    Collections.Get_From_List    ${test}    1
 163     ${stop} =    Collections.Get_From_List    ${test}    2
 164     ${ellapsed} =    Collections.Get_From_List    ${test}    3
 165     BuiltIn.Log    DATA REQUEST RESULT: Device=${number} StartTime=${start} StopTime=${stop} EllapsedTime=${ellapsed}
 166     ${data} =    Collections.Get_From_List    ${test}    4
 167     IF    '${IS_KARAF_APPL}' == 'False'
 168         ${expected} =    BuiltIn.Set_Variable    '<data xmlns="${ODL_NETCONF_NAMESPACE}"/>'
 169     ELSE
 170         ${expected} =    BuiltIn.Set_Variable    '<data xmlns="${ODL_NETCONF_NAMESPACE}"></data>'
 171     END
 172     Should Be Equal As Strings    ${data}    ${expected}
 173
 174 Check_Device_Deconfigured
 175     [Documentation]    Operation for making sure the device is really deconfigured.
 176     [Arguments]    ${current_name}    ${log_response}=True
 177     KarafKeywords.Log_Message_To_Controller_Karaf    Waiting for device ${current_name} to disappear
 178     NetconfKeywords.Wait_Device_Fully_Removed
 179     ...    ${current_name}
 180     ...    period=0.5s
 181     ...    timeout=120s
 182     ...    log_response=${log_response}
 183     KarafKeywords.Log_Message_To_Controller_Karaf    Device ${current_name} removed
 184
 185 Get Juniper Device Schemas
 186     OperatingSystem.Run    git clone https://github.com/Juniper/yang.git
 187     OperatingSystem.Run    mkdir /tmp/junos_19.4R1
 188     OperatingSystem.Run    find yang/19.4/19.4R1/junos -type f -name '*yang' -exec cp {} /tmp/junos_19.4R1/ \\;
 189     OperatingSystem.Run    cp yang/19.4/19.4R1/common/* /tmp/junos_19.4R1/
 190     OperatingSystem.List Directory    /tmp/junos_19.4R1/
 191     RETURN    /tmp/junos_19.4R1