csit/suites/controller/benchmark/dsbenchmark.robot

   1 *** Settings ***
   2 Documentation       MD-SAL Data Store benchmarking.
   3 ...
   4 ...                 Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...                 This program and the accompanying materials are made available under the
   7 ...                 terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...                 and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...                 This test suite uses the odl-dsbenchmark-impl feature controlled
  11 ...                 via dsbenchmark.py tool for testing the MD-SAL Data Store performance.
  12 ...                 (see the 'https://wiki.opendaylight.org/view/Controller_Core_Functionality_Tutorials:Tutorials:Data_Store_Benchmarking_and_Data_Access_Patterns')
  13 ...
  14 ...                 Based on values in test suite variables it triggers required numbers of
  15 ...                 warm-up and measured test runs: odl-dsbenchmark-impl module generates
  16 ...                 (towards MD-SAL Data Store) specified structure, type and number of operations.
  17 ...                 The test suite performs checks for start-up and test execution timeouts
  18 ...                 (Start Measurement, Wait For Results) and basic checks for test runs results
  19 ...                 (Check Results). Finally it provides total numbers per operation structure and type
  20 ...                 (by default in the perf_per_struct.csv, perf_per_ops.csv files)
  21 ...                 suitable for plotting in system test environment. See also the
  22 ...                 'https://wiki.opendaylight.org/view/CrossProject:Integration_Group:System_Test:Step_by_Step_Guide#Optional_-_Plot_a_graph_from_your_job'
  23 ...                 Included totals can be filtered using the FILTER parameter (RegExp).
  24 ...                 Because of the way how graphs are drawn, it is recomended to keep
  25 ...                 all test suite variables unchanged as defined for the 1st build.
  26 ...                 Parameters WARMUPS, RUNS and accordingly the TIMEOUT value can be changed
  27 ...                 for each build if needed. Parameter UNITS defines time units returned
  28 ...                 by odl-dsbenchmark-impl module. The dsbenchmark.py tool always returns
  29 ...                 values in miliseconds.
  30 ...
  31 ...                 When running this robot suite always use --exclude tag for distinguish
  32 ...                 the run for 3node setup: need a benchmark for leader and follow (--exclude singlenode_setup)
  33 ...                 the run for 1node setup: no followr present (--exclude clustered_setup)
  34
  35 Library             OperatingSystem
  36 Library             SSHLibrary    timeout=10s
  37 Library             RequestsLibrary
  38 Variables           ${CURDIR}/../../../variables/Variables.py
  39 Resource            ${CURDIR}/../../../libraries/ClusterManagement.robot
  40 Resource            ${CURDIR}/../../../libraries/RemoteBash.robot
  41 Resource            ${CURDIR}/../../../libraries/SetupUtils.robot
  42 Resource            ${CURDIR}/../../../libraries/SSHKeywords.robot
  43 Resource            ${CURDIR}/../../../libraries/Utils.robot
  44 Resource            ${CURDIR}/../../../libraries/WaitForFailure.robot
  45
  46 Suite Setup         Setup_Everything
  47 Suite Teardown      Teardown_Everything
  48 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
  49 Test Teardown       SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
  50
  51
  52 *** Variables ***
  53 ${ODL_LOG_LEVEL}            info
  54 ${TX_TYPE}                  {TX-CHAINING,SIMPLE-TX}
  55 ${OP_TYPE}                  {PUT,READ,MERGE,DELETE}
  56 ${TOTAL_OPS}                100000
  57 ${OPS_PER_TX}               100000
  58 ${INNER_OPS}                100000
  59 ${WARMUPS}                  4
  60 ${RUNS}                     8
  61 ${TIMEOUT}                  3h
  62 ${FILTER}                   EXEC
  63 ${UNITS}                    microseconds
  64 ${tool}                     dsbenchmark.py
  65 ${tool_startup_timeout}     10s
  66 ${tool_log_name}            dsbenchmark.log
  67 ${tool_output_name}         test.csv
  68 ${tool_results1_name}       perf_per_struct.csv
  69 ${tool_results2_name}       perf_per_ops.csv
  70
  71
  72 *** Test Cases ***
  73 Measure_Both_Datastores_For_One_Node_Odl_Setup
  74     [Tags]    critical    singlenode_setup
  75     [Template]    Measuring_Template
  76     leader    {CONFIG,OPERATIONAL}    both_lead_    60s
  77
  78 Measure_Leader_Config_Txchain_Put
  79     [Tags]    critical    clustered_setup
  80     [Template]    Measuring_Template
  81     leader    CONFIG    conf_lead_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
  82
  83 Measure_Leader_Config_Txchain_Read
  84     [Tags]    critical    clustered_setup
  85     [Template]    Measuring_Template
  86     leader    CONFIG    conf_lead_txchain_read_    tx_type=TX-CHAINING    op_type=READ
  87
  88 Measure_Leader_Config_Txchain_Merge
  89     [Tags]    critical    clustered_setup
  90     [Template]    Measuring_Template
  91     leader    CONFIG    conf_lead_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
  92
  93 Measure_Leader_Config_Txchain_Delete
  94     [Tags]    critical    clustered_setup
  95     [Template]    Measuring_Template
  96     leader    CONFIG    conf_lead_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
  97
  98 Measure_Leader_Config_Simpletx_Put
  99     [Tags]    critical    clustered_setup
 100     [Template]    Measuring_Template
 101     leader    CONFIG    conf_lead_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 102
 103 Measure_Leader_Config_Simpletx_Read
 104     [Tags]    critical    clustered_setup
 105     [Template]    Measuring_Template
 106     leader    CONFIG    conf_lead_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 107
 108 Measure_Leader_Config_Simpletx_Merge
 109     [Tags]    critical    clustered_setup
 110     [Template]    Measuring_Template
 111     leader    CONFIG    conf_lead_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 112
 113 Measure_Leader_Config_Simpletx_Delete
 114     [Tags]    critical    clustered_setup
 115     [Template]    Measuring_Template
 116     leader    CONFIG    conf_lead_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 117
 118 Measure_Leader_Operational_Txchain_Put
 119     [Tags]    critical    clustered_setup
 120     [Template]    Measuring_Template
 121     leader    OPERATIONAL    op_lead_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 122
 123 Measure_Leader_Operational_Txchain_Read
 124     [Tags]    critical    clustered_setup
 125     [Template]    Measuring_Template
 126     leader    OPERATIONAL    op_lead_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 127
 128 Measure_Leader_Operational_Txchain_Merge
 129     [Tags]    critical    clustered_setup
 130     [Template]    Measuring_Template
 131     leader    OPERATIONAL    op_lead_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 132
 133 Measure_Leader_Operational_Txchain_Delete
 134     [Tags]    critical    clustered_setup
 135     [Template]    Measuring_Template
 136     leader    OPERATIONAL    op_lead_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 137
 138 Measure_Leader_Operational_Simpletx_Put
 139     [Tags]    critical    clustered_setup
 140     [Template]    Measuring_Template
 141     leader    OPERATIONAL    op_lead_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 142
 143 Measure_Leader_Operational_Simpletx_Read
 144     [Tags]    critical    clustered_setup
 145     [Template]    Measuring_Template
 146     leader    OPERATIONAL    op_lead_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 147
 148 Measure_Leader_Operational_Simpletx_Merge
 149     [Tags]    critical    clustered_setup
 150     [Template]    Measuring_Template
 151     leader    OPERATIONAL    op_lead_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 152
 153 Measure_Leader_Operational_Simpletx_Delete
 154     [Tags]    critical    clustered_setup
 155     [Template]    Measuring_Template
 156     leader    OPERATIONAL    op_lead_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 157
 158 Measure_Follower_Config_Txchain_Put
 159     [Tags]    critical    clustered_setup
 160     [Template]    Measuring_Template
 161     follower    CONFIG    conf_fol_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 162
 163 Measure_Follower_Config_Txchain_Read
 164     [Tags]    critical    clustered_setup
 165     [Template]    Measuring_Template
 166     follower    CONFIG    conf_fol_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 167
 168 Measure_Follower_Config_Txchain_Merge
 169     [Tags]    critical    clustered_setup
 170     [Template]    Measuring_Template
 171     follower    CONFIG    conf_fol_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 172
 173 Measure_Follower_Config_Txchain_Delete
 174     [Tags]    critical    clustered_setup
 175     [Template]    Measuring_Template
 176     follower    CONFIG    conf_fol_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 177
 178 Measure_Follower_Config_Simpletx_Put
 179     [Tags]    critical    clustered_setup
 180     [Template]    Measuring_Template
 181     follower    CONFIG    conf_fol_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 182
 183 Measure_Follower_Config_Simpletx_Read
 184     [Tags]    critical    clustered_setup
 185     [Template]    Measuring_Template
 186     follower    CONFIG    conf_fol_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 187
 188 Measure_Follower_Config_Simpletx_Merge
 189     [Tags]    critical    clustered_setup
 190     [Template]    Measuring_Template
 191     follower    CONFIG    conf_fol_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 192
 193 Measure_Follower_Config_Simpletx_Delete
 194     [Tags]    critical    clustered_setup
 195     [Template]    Measuring_Template
 196     follower    CONFIG    conf_fol_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 197
 198 Measure_Follower_Operational_Txchain_Put
 199     [Tags]    critical    clustered_setup
 200     [Template]    Measuring_Template
 201     follower    OPERATIONAL    op_fol_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 202
 203 Measure_Follower_Operational_Txchain_Read
 204     [Tags]    critical    clustered_setup
 205     [Template]    Measuring_Template
 206     follower    OPERATIONAL    op_fol_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 207
 208 Measure_Follower_Operational_Txchain_Merge
 209     [Tags]    critical    clustered_setup
 210     [Template]    Measuring_Template
 211     follower    OPERATIONAL    op_fol_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 212
 213 Measure_Follower_Operational_Txchain_Delete
 214     [Tags]    critical    clustered_setup
 215     [Template]    Measuring_Template
 216     follower    OPERATIONAL    op_fol_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 217
 218 Measure_Follower_Operational_Simpletx_Put
 219     [Tags]    critical    clustered_setup
 220     [Template]    Measuring_Template
 221     follower    OPERATIONAL    op_fol_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 222
 223 Measure_Follower_Operational_Simpletx_Read
 224     [Tags]    critical    clustered_setup
 225     [Template]    Measuring_Template
 226     follower    OPERATIONAL    op_fol_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 227
 228 Measure_Follower_Operational_Simpletx_Merge
 229     [Tags]    critical    clustered_setup
 230     [Template]    Measuring_Template
 231     follower    OPERATIONAL    op_fol_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 232
 233 Measure_Follower_Operational_Simpletx_Delete
 234     [Tags]    critical    clustered_setup
 235     [Template]    Measuring_Template
 236     follower    OPERATIONAL    op_fol_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 237
 238 Merge_Csvs_Together
 239     [Documentation]    Merge created csvs into given file necause plot plugin cannot have more
 240     ...    source files for one graph.
 241     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 242     Merge_Csv    perf_per_ops.csv
 243     Merge_Csv    perf_per_struct.csv
 244
 245
 246 *** Keywords ***
 247 Setup_Everything
 248     [Documentation]    Setup imported resources, SSH-login to mininet machine,
 249     ...    create HTTP session, put Python tool to mininet machine.
 250     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 251     SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
 252     SSHLibrary.Open_Connection    ${TOOLS_SYSTEM_IP}
 253     SSHKeywords.Flexible_Mininet_Login
 254     SSHLibrary.Put_File    ${CURDIR}/../../../../tools/mdsal_benchmark/${tool}
 255     SSHKeywords.Virtual_Env_Create
 256     SSHKeywords.Virtual_Env_Install_Package    requests
 257     ClusterManagement.Run_Karaf_Command_On_List_Or_All    log:set ${ODL_LOG_LEVEL}
 258
 259 Teardown_Everything
 260     [Documentation]    Cleaning-up
 261     SSHKeywords.Virtual_Env_Delete
 262     SSHLibrary.Close_All_Connections
 263
 264 Start_Benchmark_Tool
 265     [Documentation]    Start the benchmark tool. Check that it has been running at least for ${tool_startup_timeout} period.
 266     ...    If the script exits early, retry once after \${retry} if specified.
 267     [Arguments]    ${tested_datastore}    ${tested_node_ip}    ${warmups}    ${runs}    ${total_ops}    ${inner_ops}
 268     ...    ${tx_type}    ${ops_per_tx}    ${op_type}    ${retry}=${EMPTY}
 269     ${command}=    BuiltIn.Set_Variable
 270     ...    python ${tool} --host ${tested_node_ip} --port ${RESTCONFPORT} --warmup ${warmups} --runs ${runs} --total ${total_ops} --inner ${inner_ops} --txtype ${tx_type} --ops ${ops_per_tx} --optype ${op_type} --plot ${FILTER} --units ${UNITS} --datastore ${tested_datastore} &> ${tool_log_name}
 271     BuiltIn.Log    ${command}
 272     SSHKeywords.Virtual_Env_Activate_On_Current_Session
 273     ${output}=    SSHLibrary.Write    ${command}
 274     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error
 275     ...    Write Until Expected Output
 276     ...    ${EMPTY}
 277     ...    ${TOOLS_SYSTEM_PROMPT}
 278     ...    ${tool_startup_timeout}
 279     ...    1s
 280     BuiltIn.Log    ${status}
 281     BuiltIn.Log    ${message}
 282     IF    "${status}" != "PASS"    RETURN
 283     IF    """${retry}""" == ""    BuiltIn.Fail    Benchmark tool is not running.
 284     BuiltIn.Comment    An ugly hack for Carbon -all- jobs being slow to finish booting. FIXME: Use WUKS and open a Bug.
 285     BuiltIn.Sleep    ${retry}
 286     ${output}=    SSHLibrary.Write    ${command}
 287     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error
 288     ...    Write Until Expected Output
 289     ...    ${EMPTY}
 290     ...    ${TOOLS_SYSTEM_PROMPT}
 291     ...    ${tool_startup_timeout}
 292     ...    1s
 293     BuiltIn.Log    ${status}
 294     BuiltIn.Log    ${message}
 295     IF    "${status}" == "PASS"
 296         BuiltIn.Fail    Benchmark tool is not running.
 297     END
 298
 299 Wait_Until_Benchmark_Tool_Finish
 300     [Documentation]    Wait until the benchmark tool is finished. Fail in case of test timeout (${timeout}).
 301     ...    In order to prevent SSH session from closing due to inactivity, newline is sent every check.
 302     [Arguments]    ${timeout}
 303     BuiltIn.Wait Until Keyword Succeeds    ${timeout}    30s    BuiltIn.Run Keywords    SSHLibrary.Write    ${EMPTY}
 304     ...    AND    SSHLibrary.Read Until Prompt
 305
 306 Stop_Benchmark_Tool
 307     [Documentation]    Stop the benchmark tool. Fail if still running.
 308     SSHLibrary.Read
 309     RemoteBash.Write_Bare_Ctrl_C
 310     SSHLibrary.Read Until Prompt
 311     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session
 312
 313 Get_Log_File
 314     [Documentation]    Return and log content of the provided file.
 315     [Arguments]    ${file_name}
 316     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 317     BuiltIn.Log    ${output_log}
 318     RETURN    ${output_log}
 319
 320 Store_File_To_Robot
 321     [Documentation]    Store the provided file from the MININET to the ROBOT machine.
 322     [Arguments]    ${file_name}    ${file_prefix}
 323     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 324     BuiltIn.Log    ${output_log}
 325     OperatingSystem.Create_File    ${file_prefix}${file_name}    ${output_log}
 326
 327 Collect Logs
 328     [Documentation]    Collect logs and detailed results for debugging
 329     ${files}=    SSHLibrary.List Files In Directory    .
 330     ${tool_log}=    Get_Log_File    ${tool_log_name}
 331     ${tool_output}=    Get_Log_File    ${tool_output_name}
 332     ${tool_results1}=    Get_Log_File    ${tool_results1_name}
 333     ${tool_results2}=    Get_Log_File    ${tool_results2_name}
 334
 335 Check Results
 336     [Documentation]    Check outputs for expected content. Fail in case of unexpected content.
 337     ${tool_log}=    Get_Log_File    ${tool_log_name}
 338     BuiltIn.Should Contain    ${tool_log}    Total execution time:
 339     BuiltIn.Should Not Contain    ${tool_log}    status: NOK
 340
 341 Set_Node_Ip_For_Benchmark
 342     [Documentation]    Returns the node ip which should be tested
 343     [Arguments]    ${state}    ${tested_ds}    ${file_prefix}
 344     IF    ${NUM_ODL_SYSTEM}==1    RETURN    ${ODL_SYSTEM_1_IP}
 345     ${shard_type}=    BuiltIn.Set_Variable_If    "${tested_ds}"=="CONFIG"    config    operational
 346     ${leader}    ${followers}=    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_type=${shard_type}
 347     IF    "${state}"=="leader"    RETURN    ${ODL_SYSTEM_${leader}_IP}
 348     RETURN    ${ODL_SYSTEM_${followers}[0]_IP}
 349
 350 Measuring_Template
 351     [Documentation]    Keywork which will cover a whole banchmark.
 352     ...    If ${file_prefix} is ${Empty} we have 1 node odl.
 353     [Arguments]    ${state}    ${tested_ds}    ${file_prefix}    ${retry}=${EMPTY}    ${warmups}=${WARMUPS}    ${runs}=${RUNS}
 354     ...    ${total_ops}=${TOTAL_OPS}    ${inner_ops}=${INNER_OPS}    ${tx_type}=${TX_TYPE}    ${ops_per_tx}=${OPS_PER_TX}    ${op_type}=${OP_TYPE}
 355     ${odl_node_ip}=    Set_Node_Ip_For_Benchmark    ${state}    ${tested_ds}    ${file_prefix}
 356     Start_Benchmark_Tool    ${tested_ds}    ${odl_node_ip}    ${warmups}    ${runs}    ${total_ops}    ${inner_ops}
 357     ...    ${tx_type}    ${ops_per_tx}    ${op_type}    retry=${retry}
 358     Wait_Until_Benchmark_Tool_Finish    ${TIMEOUT}
 359     SSHLibrary.File Should Exist    ${tool_results1_name}
 360     SSHLibrary.File Should Exist    ${tool_results2_name}
 361     Check Results
 362     Store_File_To_Robot    ${tool_results1_name}    ${file_prefix}
 363     Store_File_To_Robot    ${tool_results2_name}    ${file_prefix}
 364     ${odl_node_ip_after}=    Set_Node_Ip_For_Benchmark    ${state}    ${tested_ds}    ${file_prefix}
 365     BuiltIn.Should_Be_Equal
 366     ...    ${odl_node_ip}
 367     ...    ${odl_node_ip_after}
 368     ...    Leader changed from ${odl_node_ip} to ${odl_node_ip_after} during the benchmark.
 369     [Teardown]    Stop_Measurement_And_Save_Logs
 370
 371 Stop_Measurement_And_Save_Logs
 372     Stop_Benchmark_Tool
 373     Collect Logs
 374
 375 Merge_Csv
 376     [Documentation]    Creates ${final_file} csv file from existing files in the current directory
 377     [Arguments]    ${final_file}
 378     ${final_columns}=    BuiltIn.Set_Variable    ${Empty}
 379     ${final_values}=    BuiltIn.Set_variable    ${Empty}
 380     @{csv_files}=    OperatingSystem.List_Files_In_Directory    .    *${final_file}
 381     Collections.Sort_List    ${csv_files}
 382     FOR    ${file}    IN    @{csv_files}
 383         BuiltIn.Log_To_Console    ${file}
 384         ${csv_content}=    OperatingSystem.GetFile    ${file}
 385         ${column_names}=    Get_Column_Names    ${file}    ${csv_content}
 386         ${column_values}=    String.Get_Line    ${csv_content}    1
 387         ${final_columns}=    BuiltIn.Set_Variable_If
 388         ...    "${final_columns}"=="${Empty}"
 389         ...    ${column_names}
 390         ...    ${final_columns},${column_names}
 391         ${final_values}=    BuiltIn.Set_Variable_If
 392         ...    "${final_values}"=="${Empty}"
 393         ...    ${column_values}
 394         ...    ${final_values},${column_values}
 395     END
 396     ${content}=    BuiltIn.Catenate    SEPARATOR=${\n}    ${final_columns}    ${final_values}
 397     OperatingSystem.Create_File    ${final_file}    ${content}
 398
 399 Get_Column_Names
 400     [Documentation]    Returns the first line of the given csv file. It is modified if the file name
 401     ...    indicates that it is the file from the shard follower.
 402     [Arguments]    ${file_name}    ${file_content}
 403     ${column_names}=    String.Get_Line    ${file_content}    0
 404     IF    "_fol_" not in "${file_name}"    RETURN    ${column_names}
 405     # now we have followers and FOL_ will be prepended to the column names
 406     @{columns}=    String.Split_String    ${column_names}    ,
 407     ${final_columns}=    BuiltIn.Set_Variable    ${Empty}
 408     FOR    ${column}    IN    @{columns}
 409         ${final_columns}=    BuiltIn.Set_Variable_If
 410         ...    "${final_columns}"=="${Empty}"
 411         ...    FOL_${column}
 412         ...    ${final_columns},FOL_${column}
 413     END
 414     RETURN    ${final_columns}