csit/suites/controller/benchmark/dsbenchmark.robot

   1 *** Settings ***
   2 Documentation     MD-SAL Data Store benchmarking.
   3 ...
   4 ...               Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This test suite uses the odl-dsbenchmark-impl feature controlled
  11 ...               via dsbenchmark.py tool for testing the MD-SAL Data Store performance.
  12 ...               (see the 'https://wiki.opendaylight.org/view/Controller_Core_Functionality_Tutorials:Tutorials:Data_Store_Benchmarking_and_Data_Access_Patterns')
  13 ...
  14 ...               Based on values in test suite variables it triggers required numbers of
  15 ...               warm-up and measured test runs: odl-dsbenchmark-impl module generates
  16 ...               (towards MD-SAL Data Store) specified structure, type and number of operations.
  17 ...               The test suite performs checks for start-up and test execution timeouts
  18 ...               (Start Measurement, Wait For Results) and basic checks for test runs results
  19 ...               (Check Results). Finally it provides total numbers per operation structure and type
  20 ...               (by default in the perf_per_struct.csv, perf_per_ops.csv files)
  21 ...               suitable for plotting in system test environment. See also the
  22 ...               'https://wiki.opendaylight.org/view/CrossProject:Integration_Group:System_Test:Step_by_Step_Guide#Optional_-_Plot_a_graph_from_your_job'
  23 ...               Included totals can be filtered using the FILTER parameter (RegExp).
  24 ...               Because of the way how graphs are drawn, it is recomended to keep
  25 ...               all test suite variables unchanged as defined for the 1st build.
  26 ...               Parameters WARMUPS, RUNS and accordingly the TIMEOUT value can be changed
  27 ...               for each build if needed. Parameter UNITS defines time units returned
  28 ...               by odl-dsbenchmark-impl module. The dsbenchmark.py tool always returns
  29 ...               values in miliseconds.
  30 ...
  31 ...               Note: Currently, the number of lines in one graph makes default height insufficient.
  32 ...               Workaround: See http://stackoverflow.com/questions/34180337/set-default-height-of-jenkins-plot-generated-with-plot-plugin
  33 Suite Setup       Setup_Everything
  34 Suite Teardown    Teardown_Everything
  35 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
  36 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
  37 Library           OperatingSystem
  38 Library           SSHLibrary    timeout=10s
  39 Library           RequestsLibrary
  40 Variables         ${CURDIR}/../../../variables/Variables.py
  41 Resource          ${CURDIR}/../../../libraries/ConfigViaRestconf.robot
  42 Resource          ${CURDIR}/../../../libraries/FailFast.robot
  43 Resource          ${CURDIR}/../../../libraries/KarafKeywords.robot
  44 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  45 Resource          ${CURDIR}/../../../libraries/Utils.robot
  46 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  47
  48 *** Variables ***
  49 ${ODL_LOG_LEVEL}    DEFAULT
  50 ${TX_TYPE}        {TX-CHAINING,SIMPLE-TX}
  51 ${OP_TYPE}        {PUT,MERGE,DELETE}
  52 ${TOTAL_OPS}      100000
  53 ${OPS_PER_TX}     100000
  54 ${INNER_OPS}      100000
  55 ${WARMUPS}        10
  56 ${RUNS}           10
  57 ${TIMEOUT}        3 h
  58 ${FILTER}         EXEC
  59 ${UNITS}          microseconds
  60 ${tool}           dsbenchmark.py
  61 ${tool_args}      ${EMPTY}
  62 ${tool_startup_timeout}    10s
  63 ${tool_log_name}    dsbenchmark.log
  64 ${tool_output_name}    test.csv
  65 ${tool_results1_name}    perf_per_struct.csv
  66 ${tool_results2_name}    perf_per_ops.csv
  67
  68 *** Test Cases ***
  69 Set Karaf Log Levels
  70     [Documentation]    Set Karaf log level
  71     KarafKeywords.Execute_Controller_Karaf_Command_On_Background    log:set ${ODL_LOG_LEVEL}
  72
  73 Start Measurement
  74     [Documentation]    Start the benchmark tool. Fail if test not started.
  75     [Tags]    critical
  76     Start_Benchmark_Tool
  77
  78 Wait For Results
  79     [Documentation]    Wait until results are available. Fail if timeout occures.
  80     [Tags]    critical
  81     Wait_Until_Benchmark_Tool_Finish    ${TIMEOUT}
  82     SSHLibrary.File Should Exist    ${tool_results1_name}
  83     SSHLibrary.File Should Exist    ${tool_results2_name}
  84     Store_File_To_Robot    ${tool_results1_name}
  85     Store_File_To_Robot    ${tool_results2_name}
  86
  87 Stop Measurement
  88     [Documentation]    Stop the benchmark tool (if still running)
  89     [Setup]    FailFast.Run_Even_When_Failing_Fast
  90     Stop_Benchmark_Tool
  91
  92 Collect Logs
  93     [Documentation]    Collect logs and detailed results for debugging
  94     [Setup]    FailFast.Run_Even_When_Failing_Fast
  95     ${files}=    SSHLibrary.List Files In Directory    .
  96     ${tool_log}=    Get_Log_File    ${tool_log_name}
  97     ${tool_output}=    Get_Log_File    ${tool_output_name}
  98     ${tool_results1}=    Get_Log_File    ${tool_results1_name}
  99     ${tool_results2}=    Get_Log_File    ${tool_results2_name}
 100
 101 Check Results
 102     [Documentation]    Check outputs for expected content. Fail in case of unexpected content.
 103     [Tags]    critical
 104     ${tool_log}=    Get_Log_File    ${tool_log_name}
 105     BuiltIn.Should Contain    ${tool_log}    Total execution time:
 106     BuiltIn.Should Not Contain    ${tool_log}    status: NOK
 107
 108 *** Keywords ***
 109 Setup_Everything
 110     [Documentation]    Setup imported resources, SSH-login to mininet machine,
 111     ...    create HTTP session, put Python tool to mininet machine.
 112     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 113     SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
 114     SSHLibrary.Open_Connection    ${TOOLS_SYSTEM_IP}
 115     Utils.Flexible_Mininet_Login
 116     SSHLibrary.Put_File    ${CURDIR}/../../../../tools/mdsal_benchmark/${tool}
 117
 118 Teardown_Everything
 119     [Documentation]    Cleaning-up
 120     SSHLibrary.Close_All_Connections
 121
 122 Start_Benchmark_Tool
 123     [Documentation]    Start the benchmark tool. Check that it has been running at least for ${tool_startup_timeout} period.
 124     ${command}=    BuiltIn.Set_Variable    python ${tool} --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --warmup ${WARMUPS} --runs ${RUNS} --total ${TOTAL_OPS} --inner ${INNER_OPS} --txtype ${TX_TYPE} --ops ${OPS_PER_TX} --optype ${OP_TYPE} --plot ${FILTER} --units ${UNITS} ${tool_args} &> ${tool_log_name}
 125     BuiltIn.Log    ${command}
 126     ${output}=    SSHLibrary.Write    ${command}
 127     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error    Write Until Expected Output    ${EMPTY}    ${TOOLS_SYSTEM_PROMPT}    ${tool_startup_timeout}
 128     ...    1s
 129     BuiltIn.Log    ${status}
 130     BuiltIn.Log    ${message}
 131     BuiltIn.Run Keyword If    '${status}' == 'PASS'    BuiltIn.Fail    Benchmark tool is not running
 132
 133 Wait_Until_Benchmark_Tool_Finish
 134     [Arguments]    ${timeout}
 135     [Documentation]    Wait until the benchmark tool is finished. Fail in case of test timeout (${timeout}).
 136     ...    In order to prevent SSH session from closing due to inactivity, newline is sent every check.
 137     BuiltIn.Wait Until Keyword Succeeds    ${timeout}    30s    BuiltIn.Run Keywords    SSHLibrary.Write    ${EMPTY}
 138     ...    AND    SSHLibrary.Read Until Prompt
 139
 140 Stop_Benchmark_Tool
 141     [Documentation]    Stop the benchmark tool. Fail if still running.
 142     Utils.Write_Bare_Ctrl_C
 143     SSHLibrary.Read Until Prompt
 144
 145 Get_Log_File
 146     [Arguments]    ${file_name}
 147     [Documentation]    Return and log content of the provided file.
 148     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 149     BuiltIn.Log    ${output_log}
 150     [Return]    ${output_log}
 151
 152 Store_File_To_Robot
 153     [Arguments]    ${file_name}
 154     [Documentation]    Store the provided file from the MININET to the ROBOT machine.
 155     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 156     BuiltIn.Log    ${output_log}
 157     Create File    ${file_name}    ${output_log}