csit/suites/controller/benchmark/dsbenchmark.robot

   1 *** Settings ***
   2 Documentation     MD-SAL Data Store benchmarking.
   3 ...
   4 ...               Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...               This test suite uses the odl-dsbenchmark-impl feature controlled
  11 ...               via dsbenchmark.py tool for testing the MD-SAL Data Store performance.
  12 ...               (see the 'https://wiki.opendaylight.org/view/Controller_Core_Functionality_Tutorials:Tutorials:Data_Store_Benchmarking_and_Data_Access_Patterns')
  13 ...
  14 ...               Based on values in test suite variables it triggers required numbers of
  15 ...               warm-up and measured test runs: odl-dsbenchmark-impl module generates
  16 ...               (towards MD-SAL Data Store) specified structure, type and number of operations.
  17 ...               The test suite performs checks for start-up and test execution timeouts
  18 ...               (Start Measurement, Wait For Results) and basic checks for test runs results
  19 ...               (Check Results). Finally it provides total numbers per operation structure and type
  20 ...               (by default in the perf_per_struct.csv, perf_per_ops.csv files)
  21 ...               suitable for plotting in system test environment. See also the
  22 ...               'https://wiki.opendaylight.org/view/CrossProject:Integration_Group:System_Test:Step_by_Step_Guide#Optional_-_Plot_a_graph_from_your_job'
  23 ...               Included totals can be filtered using the FILTER parameter (RegExp).
  24 ...               Because of the way how graphs are drawn, it is recomended to keep
  25 ...               all test suite variables unchanged as defined for the 1st build.
  26 ...               Parameters WARMUPS, RUNS and accordingly the TIMEOUT value can be changed
  27 ...               for each build if needed. Parameter UNITS defines time units returned
  28 ...               by odl-dsbenchmark-impl module. The dsbenchmark.py tool always returns
  29 ...               values in miliseconds.
  30 ...
  31 ...               When running this robot suite always use --exclude tag for distinguish
  32 ...               the run for 3node setup: need a benchmark for leader and follow (--exclude singlenode_setup)
  33 ...               the run for 1node setup: no followr present (--exclude clustered_setup)
  34 Suite Setup       Setup_Everything
  35 Suite Teardown    Teardown_Everything
  36 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
  37 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_And_Start_Fast_Failing_If_Test_Failed
  38 Library           OperatingSystem
  39 Library           SSHLibrary    timeout=10s
  40 Library           RequestsLibrary
  41 Variables         ${CURDIR}/../../../variables/Variables.py
  42 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  43 Resource          ${CURDIR}/../../../libraries/RemoteBash.robot
  44 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  45 Resource          ${CURDIR}/../../../libraries/SSHKeywords.robot
  46 Resource          ${CURDIR}/../../../libraries/Utils.robot
  47 Resource          ${CURDIR}/../../../libraries/WaitForFailure.robot
  48
  49 *** Variables ***
  50 ${ODL_LOG_LEVEL}    info
  51 ${TX_TYPE}        {TX-CHAINING,SIMPLE-TX}
  52 ${OP_TYPE}        {PUT,READ,MERGE,DELETE}
  53 ${TOTAL_OPS}      100000
  54 ${OPS_PER_TX}     100000
  55 ${INNER_OPS}      100000
  56 ${WARMUPS}        4
  57 ${RUNS}           8
  58 ${TIMEOUT}        3h
  59 ${FILTER}         EXEC
  60 ${UNITS}          microseconds
  61 ${tool}           dsbenchmark.py
  62 ${tool_startup_timeout}    10s
  63 ${tool_log_name}    dsbenchmark.log
  64 ${tool_output_name}    test.csv
  65 ${tool_results1_name}    perf_per_struct.csv
  66 ${tool_results2_name}    perf_per_ops.csv
  67
  68 *** Test Cases ***
  69 Measure_Both_Datastores_For_One_Node_Odl_Setup
  70     [Tags]    critical    singlenode_setup
  71     [Template]    Measuring_Template
  72     leader    {CONFIG,OPERATIONAL}    both_lead_    60s
  73
  74 Measure_Leader_Config_Txchain_Put
  75     [Tags]    critical    clustered_setup
  76     [Template]    Measuring_Template
  77     leader    CONFIG    conf_lead_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
  78
  79 Measure_Leader_Config_Txchain_Read
  80     [Tags]    critical    clustered_setup
  81     [Template]    Measuring_Template
  82     leader    CONFIG    conf_lead_txchain_read_    tx_type=TX-CHAINING    op_type=READ
  83
  84 Measure_Leader_Config_Txchain_Merge
  85     [Tags]    critical    clustered_setup
  86     [Template]    Measuring_Template
  87     leader    CONFIG    conf_lead_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
  88
  89 Measure_Leader_Config_Txchain_Delete
  90     [Tags]    critical    clustered_setup
  91     [Template]    Measuring_Template
  92     leader    CONFIG    conf_lead_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
  93
  94 Measure_Leader_Config_Simpletx_Put
  95     [Tags]    critical    clustered_setup
  96     [Template]    Measuring_Template
  97     leader    CONFIG    conf_lead_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
  98
  99 Measure_Leader_Config_Simpletx_Read
 100     [Tags]    critical    clustered_setup
 101     [Template]    Measuring_Template
 102     leader    CONFIG    conf_lead_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 103
 104 Measure_Leader_Config_Simpletx_Merge
 105     [Tags]    critical    clustered_setup
 106     [Template]    Measuring_Template
 107     leader    CONFIG    conf_lead_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 108
 109 Measure_Leader_Config_Simpletx_Delete
 110     [Tags]    critical    clustered_setup
 111     [Template]    Measuring_Template
 112     leader    CONFIG    conf_lead_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 113
 114 Measure_Leader_Operational_Txchain_Put
 115     [Tags]    critical    clustered_setup
 116     [Template]    Measuring_Template
 117     leader    OPERATIONAL    op_lead_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 118
 119 Measure_Leader_Operational_Txchain_Read
 120     [Tags]    critical    clustered_setup
 121     [Template]    Measuring_Template
 122     leader    OPERATIONAL    op_lead_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 123
 124 Measure_Leader_Operational_Txchain_Merge
 125     [Tags]    critical    clustered_setup
 126     [Template]    Measuring_Template
 127     leader    OPERATIONAL    op_lead_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 128
 129 Measure_Leader_Operational_Txchain_Delete
 130     [Tags]    critical    clustered_setup
 131     [Template]    Measuring_Template
 132     leader    OPERATIONAL    op_lead_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 133
 134 Measure_Leader_Operational_Simpletx_Put
 135     [Tags]    critical    clustered_setup
 136     [Template]    Measuring_Template
 137     leader    OPERATIONAL    op_lead_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 138
 139 Measure_Leader_Operational_Simpletx_Read
 140     [Tags]    critical    clustered_setup
 141     [Template]    Measuring_Template
 142     leader    OPERATIONAL    op_lead_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 143
 144 Measure_Leader_Operational_Simpletx_Merge
 145     [Tags]    critical    clustered_setup
 146     [Template]    Measuring_Template
 147     leader    OPERATIONAL    op_lead_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 148
 149 Measure_Leader_Operational_Simpletx_Delete
 150     [Tags]    critical    clustered_setup
 151     [Template]    Measuring_Template
 152     leader    OPERATIONAL    op_lead_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 153
 154 Measure_Follower_Config_Txchain_Put
 155     [Tags]    critical    clustered_setup
 156     [Template]    Measuring_Template
 157     follower    CONFIG    conf_fol_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 158
 159 Measure_Follower_Config_Txchain_Read
 160     [Tags]    critical    clustered_setup
 161     [Template]    Measuring_Template
 162     follower    CONFIG    conf_fol_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 163
 164 Measure_Follower_Config_Txchain_Merge
 165     [Tags]    critical    clustered_setup
 166     [Template]    Measuring_Template
 167     follower    CONFIG    conf_fol_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 168
 169 Measure_Follower_Config_Txchain_Delete
 170     [Tags]    critical    clustered_setup
 171     [Template]    Measuring_Template
 172     follower    CONFIG    conf_fol_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 173
 174 Measure_Follower_Config_Simpletx_Put
 175     [Tags]    critical    clustered_setup
 176     [Template]    Measuring_Template
 177     follower    CONFIG    conf_fol_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 178
 179 Measure_Follower_Config_Simpletx_Read
 180     [Tags]    critical    clustered_setup
 181     [Template]    Measuring_Template
 182     follower    CONFIG    conf_fol_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 183
 184 Measure_Follower_Config_Simpletx_Merge
 185     [Tags]    critical    clustered_setup
 186     [Template]    Measuring_Template
 187     follower    CONFIG    conf_fol_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 188
 189 Measure_Follower_Config_Simpletx_Delete
 190     [Tags]    critical    clustered_setup
 191     [Template]    Measuring_Template
 192     follower    CONFIG    conf_fol_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 193
 194 Measure_Follower_Operational_Txchain_Put
 195     [Tags]    critical    clustered_setup
 196     [Template]    Measuring_Template
 197     follower    OPERATIONAL    op_fol_txchain_put_    tx_type=TX-CHAINING    op_type=PUT
 198
 199 Measure_Follower_Operational_Txchain_Read
 200     [Tags]    critical    clustered_setup
 201     [Template]    Measuring_Template
 202     follower    OPERATIONAL    op_fol_txchain_read_    tx_type=TX-CHAINING    op_type=READ
 203
 204 Measure_Follower_Operational_Txchain_Merge
 205     [Tags]    critical    clustered_setup
 206     [Template]    Measuring_Template
 207     follower    OPERATIONAL    op_fol_txchain_merge_    tx_type=TX-CHAINING    op_type=MERGE
 208
 209 Measure_Follower_Operational_Txchain_Delete
 210     [Tags]    critical    clustered_setup
 211     [Template]    Measuring_Template
 212     follower    OPERATIONAL    op_fol_txchain_del_    tx_type=TX-CHAINING    op_type=DELETE
 213
 214 Measure_Follower_Operational_Simpletx_Put
 215     [Tags]    critical    clustered_setup
 216     [Template]    Measuring_Template
 217     follower    OPERATIONAL    op_fol_simpletx_put_    tx_type=SIMPLE-TX    op_type=PUT
 218
 219 Measure_Follower_Operational_Simpletx_Read
 220     [Tags]    critical    clustered_setup
 221     [Template]    Measuring_Template
 222     follower    OPERATIONAL    op_fol_simpletx_read_    tx_type=SIMPLE-TX    op_type=READ
 223
 224 Measure_Follower_Operational_Simpletx_Merge
 225     [Tags]    critical    clustered_setup
 226     [Template]    Measuring_Template
 227     follower    OPERATIONAL    op_fol_simpletx_merge_    tx_type=SIMPLE-TX    op_type=MERGE
 228
 229 Measure_Follower_Operational_Simpletx_Delete
 230     [Tags]    critical    clustered_setup
 231     [Template]    Measuring_Template
 232     follower    OPERATIONAL    op_fol_simpletx_del_    tx_type=SIMPLE-TX    op_type=DELETE
 233
 234 Merge_Csvs_Together
 235     [Documentation]    Merge created csvs into given file necause plot plugin cannot have more
 236     ...    source files for one graph.
 237     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 238     Merge_Csv    perf_per_ops.csv
 239     Merge_Csv    perf_per_struct.csv
 240
 241 *** Keywords ***
 242 Setup_Everything
 243     [Documentation]    Setup imported resources, SSH-login to mininet machine,
 244     ...    create HTTP session, put Python tool to mininet machine.
 245     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 246     SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
 247     SSHLibrary.Open_Connection    ${TOOLS_SYSTEM_IP}
 248     SSHKeywords.Flexible_Mininet_Login
 249     SSHLibrary.Put_File    ${CURDIR}/../../../../tools/mdsal_benchmark/${tool}
 250     SSHKeywords.Virtual_Env_Create
 251     SSHKeywords.Virtual_Env_Install_Package    requests
 252     ClusterManagement.Run_Karaf_Command_On_List_Or_All    log:set ${ODL_LOG_LEVEL}
 253
 254 Teardown_Everything
 255     [Documentation]    Cleaning-up
 256     SSHKeywords.Virtual_Env_Delete
 257     SSHLibrary.Close_All_Connections
 258
 259 Start_Benchmark_Tool
 260     [Arguments]    ${tested_datastore}    ${tested_node_ip}    ${warmups}    ${runs}    ${total_ops}    ${inner_ops}
 261     ...    ${tx_type}    ${ops_per_tx}    ${op_type}    ${retry}=${EMPTY}
 262     [Documentation]    Start the benchmark tool. Check that it has been running at least for ${tool_startup_timeout} period.
 263     ...    If the script exits early, retry once after \${retry} if specified.
 264     ${command}=    BuiltIn.Set_Variable    python ${tool} --host ${tested_node_ip} --port ${RESTCONFPORT} --warmup ${warmups} --runs ${runs} --total ${total_ops} --inner ${inner_ops} --txtype ${tx_type} --ops ${ops_per_tx} --optype ${op_type} --plot ${FILTER} --units ${UNITS} --datastore ${tested_datastore} &> ${tool_log_name}
 265     BuiltIn.Log    ${command}
 266     SSHKeywords.Virtual_Env_Activate_On_Current_Session
 267     ${output}=    SSHLibrary.Write    ${command}
 268     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error    Write Until Expected Output    ${EMPTY}    ${TOOLS_SYSTEM_PROMPT}    ${tool_startup_timeout}
 269     ...    1s
 270     BuiltIn.Log    ${status}
 271     BuiltIn.Log    ${message}
 272     BuiltIn.Return From Keyword If    "${status}" != "PASS"
 273     BuiltIn.Run Keyword If    """${retry}""" == ""    BuiltIn.Fail    Benchmark tool is not running.
 274     BuiltIn.Comment    An ugly hack for Carbon -all- jobs being slow to finish booting. FIXME: Use WUKS and open a Bug.
 275     BuiltIn.Sleep    ${retry}
 276     ${output}=    SSHLibrary.Write    ${command}
 277     ${status}    ${message}=    BuiltIn.Run Keyword And Ignore Error    Write Until Expected Output    ${EMPTY}    ${TOOLS_SYSTEM_PROMPT}    ${tool_startup_timeout}
 278     ...    1s
 279     BuiltIn.Log    ${status}
 280     BuiltIn.Log    ${message}
 281     BuiltIn.Run Keyword If    "${status}" == "PASS"    BuiltIn.Fail    Benchmark tool is not running.
 282
 283 Wait_Until_Benchmark_Tool_Finish
 284     [Arguments]    ${timeout}
 285     [Documentation]    Wait until the benchmark tool is finished. Fail in case of test timeout (${timeout}).
 286     ...    In order to prevent SSH session from closing due to inactivity, newline is sent every check.
 287     BuiltIn.Wait Until Keyword Succeeds    ${timeout}    30s    BuiltIn.Run Keywords    SSHLibrary.Write    ${EMPTY}
 288     ...    AND    SSHLibrary.Read Until Prompt
 289
 290 Stop_Benchmark_Tool
 291     [Documentation]    Stop the benchmark tool. Fail if still running.
 292     SSHLibrary.Read
 293     RemoteBash.Write_Bare_Ctrl_C
 294     SSHLibrary.Read Until Prompt
 295     SSHKeywords.Virtual_Env_Deactivate_On_Current_Session
 296
 297 Get_Log_File
 298     [Arguments]    ${file_name}
 299     [Documentation]    Return and log content of the provided file.
 300     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 301     BuiltIn.Log    ${output_log}
 302     [Return]    ${output_log}
 303
 304 Store_File_To_Robot
 305     [Arguments]    ${file_name}    ${file_prefix}
 306     [Documentation]    Store the provided file from the MININET to the ROBOT machine.
 307     ${output_log}=    SSHLibrary.Execute_Command    cat ${file_name}
 308     BuiltIn.Log    ${output_log}
 309     OperatingSystem.Create_File    ${file_prefix}${file_name}    ${output_log}
 310
 311 Collect Logs
 312     [Documentation]    Collect logs and detailed results for debugging
 313     ${files}=    SSHLibrary.List Files In Directory    .
 314     ${tool_log}=    Get_Log_File    ${tool_log_name}
 315     ${tool_output}=    Get_Log_File    ${tool_output_name}
 316     ${tool_results1}=    Get_Log_File    ${tool_results1_name}
 317     ${tool_results2}=    Get_Log_File    ${tool_results2_name}
 318
 319 Check Results
 320     [Documentation]    Check outputs for expected content. Fail in case of unexpected content.
 321     ${tool_log}=    Get_Log_File    ${tool_log_name}
 322     BuiltIn.Should Contain    ${tool_log}    Total execution time:
 323     BuiltIn.Should Not Contain    ${tool_log}    status: NOK
 324
 325 Set_Node_Ip_For_Benchmark
 326     [Arguments]    ${state}    ${tested_ds}    ${file_prefix}
 327     [Documentation]    Returns the node ip which should be tested
 328     BuiltIn.Return From Keyword If    ${NUM_ODL_SYSTEM}==1    ${ODL_SYSTEM_1_IP}
 329     ${shard_type}=    BuiltIn.Set_Variable_If    "${tested_ds}"=="CONFIG"    config    operational
 330     ${leader}    ${followers}=    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_type=${shard_type}
 331     BuiltIn.Return From Keyword If    "${state}"=="leader"    ${ODL_SYSTEM_${leader}_IP}
 332     BuiltIn.Return From Keyword    ${ODL_SYSTEM_@{followers}[0]_IP}
 333
 334 Measuring_Template
 335     [Arguments]    ${state}    ${tested_ds}    ${file_prefix}    ${retry}=${EMPTY}    ${warmups}=${WARMUPS}    ${runs}=${RUNS}
 336     ...    ${total_ops}=${TOTAL_OPS}    ${inner_ops}=${INNER_OPS}    ${tx_type}=${TX_TYPE}    ${ops_per_tx}=${OPS_PER_TX}    ${op_type}=${OP_TYPE}
 337     [Documentation]    Keywork which will cover a whole banchmark.
 338     ...    If ${file_prefix} is ${Empty} we have 1 node odl.
 339     ${odl_node_ip}=    Set_Node_Ip_For_Benchmark    ${state}    ${tested_ds}    ${file_prefix}
 340     Start_Benchmark_Tool    ${tested_ds}    ${odl_node_ip}    ${warmups}    ${runs}    ${total_ops}    ${inner_ops}
 341     ...    ${tx_type}    ${ops_per_tx}    ${op_type}    retry=${retry}
 342     Wait_Until_Benchmark_Tool_Finish    ${TIMEOUT}
 343     SSHLibrary.File Should Exist    ${tool_results1_name}
 344     SSHLibrary.File Should Exist    ${tool_results2_name}
 345     Check Results
 346     Store_File_To_Robot    ${tool_results1_name}    ${file_prefix}
 347     Store_File_To_Robot    ${tool_results2_name}    ${file_prefix}
 348     ${odl_node_ip_after}=    Set_Node_Ip_For_Benchmark    ${state}    ${tested_ds}    ${file_prefix}
 349     BuiltIn.Should_Be_Equal    ${odl_node_ip}    ${odl_node_ip_after}    Leader changed from ${odl_node_ip} to ${odl_node_ip_after} during the benchmark.
 350     [Teardown]    Stop_Measurement_And_Save_Logs
 351
 352 Stop_Measurement_And_Save_Logs
 353     Stop_Benchmark_Tool
 354     Collect Logs
 355
 356 Merge_Csv
 357     [Arguments]    ${final_file}
 358     [Documentation]    Creates ${final_file} csv file from existing files in the current directory
 359     ${final_columns}=    BuiltIn.Set_Variable    ${Empty}
 360     ${final_values}=    BuiltIn.Set_variable    ${Empty}
 361     @{csv_files}=    OperatingSystem.List_Files_In_Directory    .    *${final_file}
 362     Collections.Sort_List    ${csv_files}
 363     FOR    ${file}    IN    @{csv_files}
 364         BuiltIn.Log_To_Console    ${file}
 365         ${csv_content}=    OperatingSystem.GetFile    ${file}
 366         ${column_names}=    Get_Column_Names    ${file}    ${csv_content}
 367         ${column_values}=    String.Get_Line    ${csv_content}    1
 368         ${final_columns}=    BuiltIn.Set_Variable_If    "${final_columns}"=="${Empty}"    ${column_names}    ${final_columns},${column_names}
 369         ${final_values}=    BuiltIn.Set_Variable_If    "${final_values}"=="${Empty}"    ${column_values}    ${final_values},${column_values}
 370     END
 371     ${content}=    BuiltIn.Catenate    SEPARATOR=${\n}    ${final_columns}    ${final_values}
 372     OperatingSystem.Create_File    ${final_file}    ${content}
 373
 374 Get_Column_Names
 375     [Arguments]    ${file_name}    ${file_content}
 376     [Documentation]    Returns the first line of the given csv file. It is modified if the file name
 377     ...    indicates that it is the file from the shard follower.
 378     ${column_names}=    String.Get_Line    ${file_content}    0
 379     BuiltIn.Return_From_Keyword_If    "_fol_" not in "${file_name}"    ${column_names}
 380     # now we have followers and FOL_ will be prepended to the column names
 381     @{columns}    String.Split_String    ${column_names}    ,
 382     ${final_columns}    BuiltIn.Set_Variable    ${Empty}
 383     FOR    ${column}    IN    @{columns}
 384         ${final_columns}    BuiltIn.Set_Variable_If    "${final_columns}"=="${Empty}"    FOL_${column}    ${final_columns},FOL_${column}
 385     END
 386     BuiltIn.Return_From_Keyword    ${final_columns}