New bulkomatic performance test single instance

- Add test to push 100K flows in 100 switches and plot flow stats
- Add KW in ClusterManagement for writing to karaf console
- Adjust bulk json to push multipe flows per transaction
- Add persistence off
- Add JVM monitoring

Change-Id: Ib6958bc1caae5a9e25048a16025120479d16b102
Signed-off-by: Luis Gomez <ecelgp@gmail.com>

diff --git a/csit/configplans/openflowplugin-bulkomatic-perf-daily.txt b/csit/configplans/openflowplugin-bulkomatic-perf-daily.txt
new file mode 100644 (file)
index 0000000..4987859
--- /dev/null
+++ b/csit/configplans/openflowplugin-bulkomatic-perf-daily.txt
@@ -0,0 +1,2 @@
+# Place the suites in run order:
+integration/test/csit/scripts/set_persistence_off.sh

diff --git a/csit/libraries/BulkomaticKeywords.robot b/csit/libraries/BulkomaticKeywords.robot
index fa708960b86465863838cff8014e45653b2845ce..d39090b90e29400c4c3cd0ce7a941a8b92ecd27e 100644 (file)
--- a/csit/libraries/BulkomaticKeywords.robot
+++ b/csit/libraries/BulkomaticKeywords.robot
@@ -95,6 +95,9 @@ Set DPN And Flow Count In Json Add
     ${get_string}=    Set Variable    "sal-bulk-flow:flows-per-dpn" : "1000"
     ${put_string}=    Set Variable    "sal-bulk-flow:flows-per-dpn" : "${flows_count}"
     ${json_body_add}    Replace String Using Regexp    ${str}    ${get_string}    ${put_string}
+    ${get_string}=    Set Variable    "sal-bulk-flow:batch-size" : "1"
+    ${put_string}=    Set Variable    "sal-bulk-flow:batch-size" : "${flows_count}"
+    ${json_body_add}    Replace String Using Regexp    ${json_body_add}    ${get_string}    ${put_string}
     Log    ${json_body_add}
     [Return]    ${json_body_add}
 
@@ -121,5 +124,8 @@ Set DPN And Flow Count In Json Del
     ${get_string}=    Set Variable    "sal-bulk-flow:flows-per-dpn" : "1000"
     ${put_string}=    Set Variable    "sal-bulk-flow:flows-per-dpn" : "${flows_count}"
     ${json_body_del}    Replace String Using Regexp    ${str}    ${get_string}    ${put_string}
+    ${get_string}=    Set Variable    "sal-bulk-flow:batch-size" : "1"
+    ${put_string}=    Set Variable    "sal-bulk-flow:batch-size" : "${flows_count}"
+    ${json_body_del}    Replace String Using Regexp    ${json_body_del}    ${get_string}    ${put_string}
     Log    ${json_body_del}
     [Return]    ${json_body_del}

diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot
index 7a82c346beaa9f5c9b41431d414cace5e1be1358..153be611a1ab4e00f7c52ba7955f018dcfad4b42 100644 (file)
--- a/csit/libraries/ClusterManagement.robot
+++ b/csit/libraries/ClusterManagement.robot
@@ -308,6 +308,14 @@ Run_Command_On_List_Or_All
     : FOR    ${index}    IN    @{index_list}
     \    Run_Command_On_Member    command=${command}    member_index=${index}
 
+Run_Karaf_Command_On_List_Or_All
+    [Arguments]    ${command}    ${member_index_list}=${EMPTY}
+    [Documentation]    Cycle through indices (or all), run karaf command on each.
+    ${index_list} =    ClusterManagement__Given_Or_Internal_Index_List    given_list=${member_index_list}
+    : FOR    ${index}    IN    @{index_list}
+    \    ${member_ip} =    Collections.Get_From_Dictionary    dictionary=${ClusterManagement__index_to_ip_mapping}    key=${index}
+    \    KarafKeywords.Issue Command On Karaf Console    ${command}    ${member_ip}
+
 Run_Command_On_Member
     [Arguments]    ${command}    ${member_index}
     [Documentation]    Obtain IP, call Utils and return output. This does not preserve active ssh session.

diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot
index f249a5e7ecddb3e90bfc812198e4d59ff39f2360..847149adcef0072b83cf408287812033784c3b51 100644 (file)
--- a/csit/libraries/ClusterOpenFlow.robot
+++ b/csit/libraries/ClusterOpenFlow.robot
@@ -1,6 +1,7 @@
 *** Settings ***
 Documentation     Cluster OpenFlow library. So far this library is only to be used by OpenFlow cluster test as it is very specific for this test.
 Library           RequestsLibrary
+Library           ${CURDIR}/ScaleClient.py
 Resource          ClusterManagement.robot
 Resource          MininetKeywords.robot
 Resource          Utils.robot
@@ -119,3 +120,15 @@ Take OpenFlow Device Link Up and Verify
     Wait Until Keyword Succeeds    5s    1s    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All    uri=${operational_port_1}    dictionary=${dictionary}    member_index_list=${controller_index_list}
     ${dictionary}    Create Dictionary    openflow:1=21    openflow:2=19    openflow:3=19
     Wait Until Keyword Succeeds    5s    1s    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All    uri=${OPERATIONAL_TOPO_API}    dictionary=${dictionary}    member_index_list=${controller_index_list}
+
+Verify_Switch_Connections_Running_On_Member
+    [Arguments]    ${switch_count}    ${member_index}
+    [Documentation]    Check if number of Switch connections on member of given index is equal to ${switch_count}.
+    ${count} =    ScaleClient.Get_Switches_Count    controller=${ODL_SYSTEM_${member_index}_IP}
+    BuiltIn.Should_Be_Equal_As_Numbers    ${switch_count}    ${count}
+
+Check_Flows_Operational_Datastore_On_Member
+    [Arguments]    ${flow_count}    ${member_index}
+    [Documentation]    Check if number of Operational Flows on member of given index is equal to ${flow_count}.
+    ${sw}    ${repf}    ${found_flow}=    ScaleClient.Flow Stats Collected    controller=${ODL_SYSTEM_${member_index}_IP}
+    BuiltIn.Should_Be_Equal_As_Numbers    ${flow_count}    ${found_flow}

diff --git a/csit/scriptplans/openflowplugin-bulkomatic-perf-daily.txt b/csit/scriptplans/openflowplugin-bulkomatic-perf-daily.txt
new file mode 100644 (file)
index 0000000..1a31c3a
--- /dev/null
+++ b/csit/scriptplans/openflowplugin-bulkomatic-perf-daily.txt
@@ -0,0 +1,3 @@
+# Place the scripts in run order:
+integration/test/csit/scripts/set_elasticsearch_attribute_short.sh
+integration/test/csit/scripts/set_jvm_common_attribute.sh

diff --git a/csit/suites/openflowplugin/Bulkomatic/100K_Flows_100DPN_1Node_Perf_Test.robot b/csit/suites/openflowplugin/Bulkomatic/100K_Flows_100DPN_1Node_Perf_Test.robot
new file mode 100644 (file)
index 0000000..785c771
--- /dev/null
+++ b/csit/suites/openflowplugin/Bulkomatic/100K_Flows_100DPN_1Node_Perf_Test.robot
@@ -0,0 +1,100 @@
+*** Settings ***
+Documentation     Test suite for 3Node Cluster - 100K flows and 10 DPNs in Cluster Scale Up scenario
+Suite Setup       ClusterManagement Setup
+Suite Teardown    Delete All Sessions
+Library           DateTime
+Library           OperatingSystem
+Resource          ../../../libraries/BulkomaticKeywords.robot
+Resource          ../../../libraries/MininetKeywords.robot
+Resource          ../../../libraries/ClusterManagement.robot
+Resource          ../../../libraries/ClusterOpenFlow.robot
+Variables         ../../../variables/Variables.py
+
+*** Variables ***
+${operation_timeout}    250s
+${oper_ds_timeout}    400s
+${mininet_timeout}    120s
+${flow_count_per_switch}    1000
+${switch_count}    100
+${flow_count_after_add}    100000
+${flow_count_after_del}    0
+${karaf_log_level}    log:set ERROR
+${orig_json_config_add}    sal_add_bulk_flow_config.json
+${orig_json_config_get}    sal_get_bulk_flow_config.json
+${orig_json_config_del}    sal_del_bulk_flow_config.json
+${rate_results_file}    rate.csv
+${time_results_file}    time.csv
+
+*** Test Cases ***
+Initialize Variables
+    [Documentation]    Check Status for all shards in OpenFlow application and set the logs across cluster nodes.
+    ClusterManagement.Run_Karaf_Command_On_List_Or_All    ${karaf_log_level}
+    ${temp_json_config_add}    BulkomaticKeywords.Set DPN And Flow Count In Json Add    ${orig_json_config_add}    ${switch_count}    ${flow_count_per_switch}
+    ${temp_json_config_get}    BulkomaticKeywords.Set DPN And Flow Count In Json Get    ${orig_json_config_get}    ${switch_count}    ${flow_count_after_add}
+    ${temp_json_config_del}    BulkomaticKeywords.Set DPN And Flow Count In Json Del    ${orig_json_config_del}    ${switch_count}    ${flow_count_per_switch}
+    BuiltIn.Set Suite Variable    ${temp_json_config_add}
+    BuiltIn.Set Suite Variable    ${temp_json_config_get}
+    BuiltIn.Set Suite Variable    ${temp_json_config_del}
+
+Start Mininet And verify Switches
+    [Documentation]    Start mininet.
+    ${mininet_conn_id}=    MininetKeywords.Start Mininet Single Controller    ${TOOLS_SYSTEM_IP}    ${ODL_SYSTEM_IP}    --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13
+    BuiltIn.Set Suite Variable    ${mininet_conn_id}
+    BuiltIn.Wait Until Keyword Succeeds    ${mininet_timeout}    2s    ClusterOpenFlow.Verify_Switch_Connections_Running_On_Member    ${switch_count}    1
+
+Add Bulk Flow
+    [Documentation]    100K Flows (1K Flows per DPN) in 100 DPN added and verify it gets applied.
+    ${config_datastore_write_start_time}=    DateTime.Get Current Date    result_format=timestamp
+    BulkomaticKeywords.Add Bulk Flow In Node    ${temp_json_config_add}    1    ${operation_timeout}
+    ${config_datastore_write_end_time}=    DateTime.Get Current Date    result_format=timestamp
+    ${config_datastore_write_time}=    DateTime.Subtract Date From Date    ${config_datastore_write_end_time}    ${config_datastore_write_start_time}
+    ${config_datastore_write_rate}=    BuiltIn.Evaluate    ${flow_count_after_add} / ${config_datastore_write_time}
+    BuiltIn.Set Suite Variable    ${config_datastore_write_start_time}
+    BuiltIn.Log to console    ${\n}
+    BuiltIn.Log to console    The config_datastore_write_time is ${config_datastore_write_time}
+    BuiltIn.Log to console    The config_datastore_write_rate is ${config_datastore_write_rate}
+    BuiltIn.Set Suite Variable    ${config_datastore_write_time}
+    BuiltIn.Set Suite Variable    ${config_datastore_write_rate}
+
+Verify Flows In Switch
+    [Documentation]    Verify 100K flows are installed in 100 switches.
+    ${Mininet_write_start_time}=    DateTime.Get Current Date    result_format=timestamp
+    MininetKeywords.Verify Aggregate Flow From Mininet Session    ${mininet_conn_id}    ${switch_count}    ${flow_count_per_switch}    ${operation_timeout}
+    ${Mininet_write_end_time}=    DateTime.Get Current Date    result_format=timestamp
+    ${Mininet_write_time}=    DateTime.Subtract Date From Date    ${Mininet_write_end_time}    ${config_datastore_write_start_time}
+    ${Mininet_write_rate}=    BuiltIn.Evaluate    ${flow_count_after_add} / ${Mininet_write_time}
+    BuiltIn.Log to console    ${\n}
+    BuiltIn.Log to console    The Mininet_write_time is ${Mininet_write_time}
+    BuiltIn.Log to console    The Mininet_write_rate is ${Mininet_write_rate}
+    BuiltIn.Set Suite Variable    ${Mininet_write_time}
+    BuiltIn.Set Suite Variable    ${Mininet_write_rate}
+
+Verify Flows In Oper DS
+    [Documentation]    Check Flows in Operational Datastore
+    BuiltIn.Wait Until Keyword Succeeds    ${oper_ds_timeout}    2s    ClusterOpenFlow.Check_Flows_Operational_Datastore_On_Member    ${flow_count_after_add}    1
+    ${oper_datastore_write_end_time}=    DateTime.Get Current Date    result_format=timestamp
+    ${oper_datastore_write_time}=    DateTime.Subtract Date From Date    ${oper_datastore_write_end_time}    ${config_datastore_write_start_time}
+    ${oper_datastore_write_rate}=    BuiltIn.Evaluate    ${flow_count_after_add} / ${oper_datastore_write_time}
+    BuiltIn.Log to console    ${\n}
+    BuiltIn.Log to console    The operational_datastore_write_time is ${oper_datastore_write_time}
+    BuiltIn.Log to console    The operational_datastore_write_rate is ${oper_datastore_write_rate}
+    BuiltIn.Set Suite Variable    ${oper_datastore_write_time}
+    BuiltIn.Set Suite Variable    ${oper_datastore_write_rate}
+
+Stop Mininet And Verify
+    [Documentation]    Stop mininet and exit connection.
+    MininetKeywords.Stop Mininet And Exit    ${mininet_conn_id}
+    BuiltIn.Wait Until Keyword Succeeds    ${mininet_timeout}    2s    ClusterOpenFlow.Verify_Switch_Connections_Running_On_Member    0    1
+
+Delete All Flows
+    [Documentation]    100K Flows deleted and verify.
+    ${config_datastore_delete_start_time}=    DateTime.Get Current Date    result_format=timestamp
+    BulkomaticKeywords.Delete Bulk Flow In Node    ${temp_json_config_del}    1    ${operation_timeout}
+    BuiltIn.Set Suite Variable    ${config_datastore_delete_start_time}
+
+Log Results And Determine Status
+    [Documentation]    Log results for plotting.
+    OperatingSystem.Append To File    ${rate_results_file}    Config DS,OVS Switch,Operatioanl DS\n
+    OperatingSystem.Append To File    ${rate_results_file}    ${config_datastore_write_rate},${Mininet_write_rate},${oper_datastore_write_rate}\n
+    OperatingSystem.Append To File    ${time_results_file}    Config DS,OVS Switch,Operatioanl DS\n
+    OperatingSystem.Append To File    ${time_results_file}    ${config_datastore_write_time},${Mininet_write_time},${oper_datastore_write_time}\n

diff --git a/csit/testplans/openflowplugin-bulkomatic-perf-daily.txt b/csit/testplans/openflowplugin-bulkomatic-perf-daily.txt
new file mode 100644 (file)
index 0000000..9d264fe
--- /dev/null
+++ b/csit/testplans/openflowplugin-bulkomatic-perf-daily.txt
@@ -0,0 +1,3 @@
+# Place the suites in run order:
+integration/test/csit/suites/openflowplugin/Bulkomatic/100K_Flows_100DPN_1Node_Perf_Test.robot
+integration/test/csit/suites/integration/Create_JVM_Plots.robot