csit/suites/controller/benchmark/dsbenchmark.robot

   1 *** Settings ***
   2 Documentation     MD-SAL Data Store benchmarking.
   3 ...
   4 ...               Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This test suite uses the odl-dsbenchmark-impl feature controlled
  11 ...               via dsbenchmark.py tool for testing the MD-SAL Data Store performance.
  12 ...               (see the 'https://wiki.opendaylight.org/view/Controller_Core_Functionality_Tutorials:Tutorials:Data_Store_Benchmarking_and_Data_Access_Patterns')
  13 ...
  14 ...               Based on values in test suite variables it triggers required numbers of
  15 ...               warm-up and measured test runs: odl-dsbenchmark-impl module generates
  16 ...               (towards MD-SAL Data Store) specified structure, type and number of operations.
  17 ...               The test suite performs checks for start-up and test execution timeouts
  18 ...               (Start Measurement, Wait For Results) and basic checks for test runs results
  19 ...               (Check Results). Finally it provides total numbers per operation structure and type
  20 ...               (by default in the perf_per_struct.csv, perf_per_ops.csv files)
  21 ...               suitable for plotting in system test environment. See also the
  22 ...               'https://wiki.opendaylight.org/view/CrossProject:Integration_Group:System_Test:Step_by_Step_Guide#Optional_-_Plot_a_graph_from_your_job'
  23 ...               Included totals can be filtered using the FILTER parameter (RegExp).
  24 ...               Because of the way how graphs are drawn, it is recomended to keep
  25 ...               all test suite variables unchanged as defined for the 1st build.
  26 ...               Parameters WARMUPS, RUNS and accordingly the TIMEOUT value can be changed
  27 ...               for each build if needed. Parameter UNITS defines time units returned
  28 ...               by odl-dsbenchmark-impl module. The dsbenchmark.py tool always returns
  29 ...               values in miliseconds.
  30 ...
  31 ...               When running this robot suite always use --exclude tag for distinguish
  32 ...               the run for 3node setup: need a benchmark for leader and follow (--exclude singlenode_setup)
  33 ...               the run for 1node setup: no followr present (--exclude clustered_setup)
  34 Suite Setup       Setup_Everything
  35 Suite Teardown    Teardown_Everything
  36 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
  37 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
  38 Library           OperatingSystem
  39 Library           SSHLibrary    timeout=10s
  40 Library           RequestsLibrary
  41 Variables         ${CURDIR}/../../../variables/Variables.py
  42 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  43 Resource          ${CURDIR}/../../../libraries/RemoteBash.robot
  44 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  45 Resource          ${CURDIR}/../../../libraries/Utils.robot
  46 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  47
  48 *** Variables ***
  49 ${ODL_LOG_LEVEL}    info
  50 ${TX_TYPE}        {TX-CHAINING,SIMPLE-TX}
  51 ${OP_TYPE}        {PUT,READ,MERGE,DELETE}
  52 ${TOTAL_OPS}      100000
  53 ${OPS_PER_TX}     100000
  54 ${INNER_OPS}      100000
  55 ${WARMUPS}        4
  56 ${RUNS}           8
  57 ${TIMEOUT}        3h
  58 ${FILTER}         EXEC
  59 ${UNITS}          microseconds
  60 ${tool}           dsbenchmark.py
  61 ${tool_startup_timeout}    10s
  62 ${tool_log_name}    dsbenchmark.log
  63 ${tool_output_name}    test.csv
  64 ${tool_results1_name}    perf_per_struct.csv
  65 ${tool_results2_name}    perf_per_ops.csv
  66
  67 *** Test Cases ***
  68 Measure_Both_Datastores_For_One_Node_Odl_Setup
  69     [Tags]    critical    singlenode_setup
  70     [Template]    Measuring_Template
  71     leader    {CONFIG,OPERATIONAL}    both_lead_    60s
  72
  73 Measure_Leader_Config_Txchain_Put
  74     [Tags]    critical    clustered_setup
  75     [Template]    Measuring_Template
  76     leader    CONFIG    conf_lead_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
  77
  78 Measure_Leader_Config_Txchain_Read
  79     [Tags]    critical    clustered_setup
  80     [Template]    Measuring_Template
  81     leader    CONFIG    conf_lead_txchain_read_    tx_type=TX-CHAINING    op_type=READ
  82
  83 Measure_Leader_Config_Txchain_Merge
  84     [Tags]    critical    clustered_setup
  85     [Template]    Measuring_Template
  86     leader    CONFIG    conf_lead_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
  87
  88 Measure_Leader_Config_Txchain_Delete
  89     [Tags]    critical    clustered_setup
  90     [Template]    Measuring_Template
  91     leader    CONFIG    conf_lead_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
  92
  93 Measure_Leader_Config_Simpletx_Put
  94     [Tags]    critical    clustered_setup
  95     [Template]    Measuring_Template
  96     leader    CONFIG    conf_lead_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
  97
  98 Measure_Leader_Config_Simpletx_Read
  99     [Tags]    critical    clustered_setup
 100     [Template]    Measuring_Template
 101     leader    CONFIG    conf_lead_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 102
 103 Measure_Leader_Config_Simpletx_Merge
 104     [Tags]    critical    clustered_setup
 105     [Template]    Measuring_Template
 106     leader    CONFIG    conf_lead_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 107
 108 Measure_Leader_Config_Simpletx_Delete
 109     [Tags]    critical    clustered_setup
 110     [Template]    Measuring_Template
 111     leader    CONFIG    conf_lead_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 112
 113 Measure_Leader_Operational_Txchain_Put
 114     [Tags]    critical    clustered_setup
 115     [Template]    Measuring_Template
 116     leader    OPERATIONAL    op_lead_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 117
 118 Measure_Leader_Operational_Txchain_Read
 119     [Tags]    critical    clustered_setup
 120     [Template]    Measuring_Template
 121     leader    OPERATIONAL    op_lead_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 122
 123 Measure_Leader_Operational_Txchain_Merge
 124     [Tags]    critical    clustered_setup
 125     [Template]    Measuring_Template
 126     leader    OPERATIONAL    op_lead_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 127
 128 Measure_Leader_Operational_Txchain_Delete
 129     [Tags]    critical    clustered_setup
 130     [Template]    Measuring_Template
 131     leader    OPERATIONAL    op_lead_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 132
 133 Measure_Leader_Operational_Simpletx_Put
 134     [Tags]    critical    clustered_setup
 135     [Template]    Measuring_Template
 136     leader    OPERATIONAL    op_lead_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 137
 138 Measure_Leader_Operational_Simpletx_Read
 139     [Tags]    critical    clustered_setup
 140     [Template]    Measuring_Template
 141     leader    OPERATIONAL    op_lead_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 142
 143 Measure_Leader_Operational_Simpletx_Merge
 144     [Tags]    critical    clustered_setup
 145     [Template]    Measuring_Template
 146     leader    OPERATIONAL    op_lead_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 147
 148 Measure_Leader_Operational_Simpletx_Delete
 149     [Tags]    critical    clustered_setup
 150     [Template]    Measuring_Template
 151     leader    OPERATIONAL    op_lead_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 152
 153 Measure_Follower_Config_Txchain_Put
 154     [Tags]    critical    clustered_setup
 155     [Template]    Measuring_Template
 156     follower    CONFIG    conf_fol_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 157
 158 Measure_Follower_Config_Txchain_Read
 159     [Tags]    critical    clustered_setup
 160     [Template]    Measuring_Template
 161     follower    CONFIG    conf_fol_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 162
 163 Measure_Follower_Config_Txchain_Merge
 164     [Tags]    critical    clustered_setup
 165     [Template]    Measuring_Template
 166     follower    CONFIG    conf_fol_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 167
 168 Measure_Follower_Config_Txchain_Delete
 169     [Tags]    critical    clustered_setup
 170     [Template]    Measuring_Template
 171     follower    CONFIG    conf_fol_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 172
 173 Measure_Follower_Config_Simpletx_Put
 174     [Tags]    critical    clustered_setup
 175     [Template]    Measuring_Template
 176     follower    CONFIG    conf_fol_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 177
 178 Measure_Follower_Config_Simpletx_Read
 179     [Tags]    critical    clustered_setup
 180     [Template]    Measuring_Template
 181     follower    CONFIG    conf_fol_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 182
 183 Measure_Follower_Config_Simpletx_Merge
 184     [Tags]    critical    clustered_setup
 185     [Template]    Measuring_Template
 186     follower    CONFIG    conf_fol_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 187
 188 Measure_Follower_Config_Simpletx_Delete
 189     [Tags]    critical    clustered_setup
 190     [Template]    Measuring_Template
 191     follower    CONFIG    conf_fol_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 192
 193 Measure_Follower_Operational_Txchain_Put
 194     [Tags]    critical    clustered_setup
 195     [Template]    Measuring_Template
 196     follower    OPERATIONAL    op_fol_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 197
 198 Measure_Follower_Operational_Txchain_Read
 199     [Tags]    critical    clustered_setup
 200     [Template]    Measuring_Template
 201     follower    OPERATIONAL    op_fol_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 202
 203 Measure_Follower_Operational_Txchain_Merge
 204     [Tags]    critical    clustered_setup
 205     [Template]    Measuring_Template
 206     follower    OPERATIONAL    op_fol_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 207
 208 Measure_Follower_Operational_Txchain_Delete
 209     [Tags]    critical    clustered_setup
 210     [Template]    Measuring_Template
 211     follower    OPERATIONAL    op_fol_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 212
 213 Measure_Follower_Operational_Simpletx_Put
 214     [Tags]    critical    clustered_setup
 215     [Template]    Measuring_Template
 216     follower    OPERATIONAL    op_fol_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 217
 218 Measure_Follower_Operational_Simpletx_Read
 219     [Tags]    critical    clustered_setup
 220     [Template]    Measuring_Template
 221     follower    OPERATIONAL    op_fol_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 222
 223 Measure_Follower_Operational_Simpletx_Merge
 224     [Tags]    critical    clustered_setup
 225     [Template]    Measuring_Template
 226     follower    OPERATIONAL    op_fol_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 227
 228 Measure_Follower_Operational_Simpletx_Delete
 229     [Tags]    critical    clustered_setup
 230     [Template]    Measuring_Template
 231     follower    OPERATIONAL    op_fol_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 232
 233 Merge_Csvs_Together
 234     [Documentation]    Merge created csvs into given file necause plot plugin cannot have more
 235     ...    source files for one graph.
 236     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 237     Merge_Csv    perf_per_ops.csv
 238     Merge_Csv    perf_per_struct.csv
 239
 240 *** Keywords ***
 241 Setup_Everything
 242     [Documentation]    Setup imported resources, SSH-login to mininet machine,
 243     ...    create HTTP session, put Python tool to mininet machine.
 244     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 245     SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
 246     SSHLibrary.Open_Connection    ${TOOLS_SYSTEM_IP}
 247     Utils.Flexible_Mininet_Login
 248     SSHLibrary.Put_File    ${CURDIR}/../../../../tools/mdsal_benchmark/${tool}
 249     SSHKeywords.Virtual_Env_Create
 250     SSHKeywords.Virtual_Env_Install_Package    requests
 251     ClusterManagement.Run_Karaf_Command_On_List_Or_All    log:set ${ODL_LOG_LEVEL}
 252
 253 Teardown_Everything
 254     [Documentation]    Cleaning-up
 255     SSHKeywords.Virtual_Env_Delete
 256     SSHLibrary.Close_All_Connections
 257
 258 Start_Benchmark_Tool
 259     [Arguments]    ${tested_datastore}    ${tested_node_ip}    ${warmups}    ${runs}    ${total_ops}    ${inner_ops}
 260     ...    ${tx_type}    ${ops_per_tx}    ${op_type}    ${retry}=${EMPTY}
 261     [Documentation]    Start the benchmark tool. Check that it has been running at least for ${tool_startup_timeout} period.
 262     ...    If the script exits early, retry once after \${retry} if specified.
 263     ${command}=    BuiltIn.Set_Variable    python ${tool} --host ${tested_node_ip} --port ${RESTCONFPORT} --warmup ${warmups} --runs ${runs} --total ${total_ops} --inner ${inner_ops} --txtype ${tx_type} --ops ${ops_per_tx} --optype ${op_type} --plot ${FILTER} --units ${UNITS} --datastore ${tested_datastore} &> ${tool_log_name}
 264     BuiltIn.Log    ${command}
 265     SSHKeywords.Virtual_Env_Activate_On_Current_Session
 266     ${output}=    SSHLibrary.Write    ${command}
 267     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error    Write Until Expected Output    ${EMPTY}    ${TOOLS_SYSTEM_PROMPT}    ${tool_startup_timeout}
 268     ...    1s
 269     BuiltIn.Log    ${status}
 270     BuiltIn.Log    ${message}
 271     BuiltIn.Return From Keyword If    "${status}" != "PASS"
 272     BuiltIn.Run Keyword If    """${retry}""" == ""    BuiltIn.Fail    Benchmark tool is not running.
 273     BuiltIn.Comment    An ugly hack for Carbon -all- jobs being slow to finish booting. FIXME: Use WUKS and open a Bug.
 274     BuiltIn.Sleep    ${retry}
 275     ${output}=    SSHLibrary.Write    ${command}
 276     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error    Write Until Expected Output    ${EMPTY}    ${TOOLS_SYSTEM_PROMPT}    ${tool_startup_timeout}
 277     ...    1s
 278     BuiltIn.Log    ${status}
 279     BuiltIn.Log    ${message}
 280     BuiltIn.Run Keyword If    "${status}" == "PASS"    BuiltIn.Fail    Benchmark tool is not running.
 281
 282 Wait_Until_Benchmark_Tool_Finish
 283     [Arguments]    ${timeout}
 284     [Documentation]    Wait until the benchmark tool is finished. Fail in case of test timeout (${timeout}).
 285     ...    In order to prevent SSH session from closing due to inactivity, newline is sent every check.
 286     BuiltIn.Wait Until Keyword Succeeds    ${timeout}    30s    BuiltIn.Run Keywords    SSHLibrary.Write    ${EMPTY}
 287     ...    AND    SSHLibrary.Read Until Prompt
 288
 289 Stop_Benchmark_Tool
 290     [Documentation]    Stop the benchmark tool. Fail if still running.
 291     SSHLibrary.Read
 292     RemoteBash.Write_Bare_Ctrl_C
 293     SSHLibrary.Read Until Prompt
 294     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session
 295
 296 Get_Log_File
 297     [Arguments]    ${file_name}
 298     [Documentation]    Return and log content of the provided file.
 299     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 300     BuiltIn.Log    ${output_log}
 301     [Return]    ${output_log}
 302
 303 Store_File_To_Robot
 304     [Arguments]    ${file_name}    ${file_prefix}
 305     [Documentation]    Store the provided file from the MININET to the ROBOT machine.
 306     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 307     BuiltIn.Log    ${output_log}
 308     OperatingSystem.Create_File    ${file_prefix}${file_name}    ${output_log}
 309
 310 Collect Logs
 311     [Documentation]    Collect logs and detailed results for debugging
 312     ${files}=    SSHLibrary.List Files In Directory    .
 313     ${tool_log}=    Get_Log_File    ${tool_log_name}
 314     ${tool_output}=    Get_Log_File    ${tool_output_name}
 315     ${tool_results1}=    Get_Log_File    ${tool_results1_name}
 316     ${tool_results2}=    Get_Log_File    ${tool_results2_name}
 317
 318 Check Results
 319     [Documentation]    Check outputs for expected content. Fail in case of unexpected content.
 320     ${tool_log}=    Get_Log_File    ${tool_log_name}
 321     BuiltIn.Should Contain    ${tool_log}    Total execution time:
 322     BuiltIn.Should Not Contain    ${tool_log}    status: NOK
 323
 324 Set_Node_Ip_For_Benchmark
 325     [Arguments]    ${state}    ${tested_ds}    ${file_prefix}
 326     [Documentation]    Returns the node ip which should be tested
 327     BuiltIn.Return From Keyword If    ${NUM_ODL_SYSTEM}==1    ${ODL_SYSTEM_1_IP}
 328     ${shard_type}=    BuiltIn.Set_Variable_If    "${tested_ds}"=="CONFIG"    config    operational
 329     ${leader}    ${followers}=    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_type=${shard_type}
 330     BuiltIn.Return From Keyword If    "${state}"=="leader"    ${ODL_SYSTEM_${leader}_IP}
 331     BuiltIn.Return From Keyword    ${ODL_SYSTEM_@{followers}[0]_IP}
 332
 333 Measuring_Template
 334     [Arguments]    ${state}    ${tested_ds}    ${file_prefix}    ${retry}=${EMPTY}    ${warmups}=${WARMUPS}    ${runs}=${RUNS}
 335     ...    ${total_ops}=${TOTAL_OPS}    ${inner_ops}=${INNER_OPS}    ${tx_type}=${TX_TYPE}    ${ops_per_tx}=${OPS_PER_TX}    ${op_type}=${OP_TYPE}
 336     [Documentation]    Keywork which will cover a whole banchmark.
 337     ...    If ${file_prefix} is ${Empty} we have 1 node odl.
 338     ${odl_node_ip}=    Set_Node_Ip_For_Benchmark    ${state}    ${tested_ds}    ${file_prefix}
 339     Start_Benchmark_Tool    ${tested_ds}    ${odl_node_ip}    ${warmups}    ${runs}    ${total_ops}    ${inner_ops}
 340     ...    ${tx_type}    ${ops_per_tx}    ${op_type}    retry=${retry}
 341     Wait_Until_Benchmark_Tool_Finish    ${TIMEOUT}
 342     SSHLibrary.File Should Exist    ${tool_results1_name}
 343     SSHLibrary.File Should Exist    ${tool_results2_name}
 344     Check Results
 345     Store_File_To_Robot    ${tool_results1_name}    ${file_prefix}
 346     Store_File_To_Robot    ${tool_results2_name}    ${file_prefix}
 347     ${odl_node_ip_after}=    Set_Node_Ip_For_Benchmark    ${state}    ${tested_ds}    ${file_prefix}
 348     BuiltIn.Should_Be_Equal    ${odl_node_ip}    ${odl_node_ip_after}    Leader changed from ${odl_node_ip} to ${odl_node_ip_after} during the benchmark.
 349     [Teardown]    Stop_Measurement_And_Save_Logs
 350
 351 Stop_Measurement_And_Save_Logs
 352     Stop_Benchmark_Tool
 353     Collect Logs
 354
 355 Merge_Csv
 356     [Arguments]    ${final_file}
 357     [Documentation]    Creates ${final_file} csv file from existing files in the current directory
 358     ${final_columns}=    BuiltIn.Set_Variable    ${Empty}
 359     ${final_values}=    BuiltIn.Set_variable    ${Empty}
 360     @{csv_files}=    OperatingSystem.List_Files_In_Directory    .    *${final_file}
 361     Collections.Sort_List    ${csv_files}
 362     : FOR    ${file}    IN    @{csv_files}
 363     \    BuiltIn.Log_To_Console    ${file}
 364     \    ${csv_content}=    OperatingSystem.GetFile    ${file}
 365     \    ${column_names}=    Get_Column_Names    ${file}    ${csv_content}
 366     \    ${column_values}=    String.Get_Line    ${csv_content}    1
 367     \    ${final_columns}=    BuiltIn.Set_Variable_If    "${final_columns}"=="${Empty}"    ${column_names}    ${final_columns},${column_names}
 368     \    ${final_values}=    BuiltIn.Set_Variable_If    "${final_values}"=="${Empty}"    ${column_values}    ${final_values},${column_values}
 369     ${content}=    BuiltIn.Catenate    SEPARATOR=${\n}    ${final_columns}    ${final_values}
 370     OperatingSystem.Create_File    ${final_file}    ${content}
 371
 372 Get_Column_Names
 373     [Arguments]    ${file_name}    ${file_content}
 374     [Documentation]    Returns the first line of the given csv file. It is modified if the file name
 375     ...    indicates that it is the file from the shard follower.
 376     ${column_names}=    String.Get_Line    ${file_content}    0
 377     BuiltIn.Return_From_Keyword_If    "_fol_" not in "${file_name}"    ${column_names}
 378     # now we have followers and FOL_ will be prepended to the column names
 379     @{columns}    String.Split_String    ${column_names}    ,
 380     ${final_columns}    BuiltIn.Set_Variable    ${Empty}
 381     : FOR    ${column}    IN    @{columns}
 382     \    ${final_columns}    BuiltIn.Set_Variable_If    "${final_columns}"=="${Empty}"    FOL_${column}    ${final_columns},FOL_${column}
 383     BuiltIn.Return_From_Keyword    ${final_columns}