Verification of OF18 current Term 86/53886/43
authornaveen <naveenk.t@tcs.com>
Mon, 10 Apr 2017 20:33:09 +0000 (02:03 +0530)
committerLuis Gomez <ecelgp@gmail.com>
Mon, 17 Jul 2017 02:13:03 +0000 (02:13 +0000)
Change-Id: I21cb25420d984db1da4f2a0d1e9a34c589afa3ce
Signed-off-by: naveen <naveenk.t@tcs.com>
csit/libraries/BulkomaticKeywords.robot
csit/libraries/ClusterManagement.robot
csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot [new file with mode: 0644]
csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt [new file with mode: 0644]
csit/variables/openflowplugin/add_table.json [new file with mode: 0644]
csit/variables/openflowplugin/sal_table_add.json [new file with mode: 0644]
csit/variables/openflowplugin/sal_table_add_config.json [new file with mode: 0644]
csit/variables/openflowplugin/sal_table_flow_config.json [new file with mode: 0644]

index c76bc7e75ad46d3138d9b8740f08573dbbcd8ada..84e3697112b7fc01c7afb0111ec8c71dd76177d5 100644 (file)
@@ -8,6 +8,7 @@ Variables         ../variables/Variables.py
 *** Variables ***
 ${ADD_BULK_CONFIG_NODES_API}    /restconf/operations/sal-bulk-flow:flow-test
 ${GET_BULK_CONFIG_NODES_API}    /restconf/operations/sal-bulk-flow:read-flow-test
+${ADD_TABLE_NODEs_API}    /restconf/operations/sal-bulk-flow:table-test
 ${jolokia_write_op_status}    /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/WriteOpStatus
 ${jolokia_read_op_status}    /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/ReadOpStatus
 ${jolokia_flow_count_status}    /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/FlowCount
@@ -38,6 +39,11 @@ Add Bulk Flow
     [Documentation]    Add Bulk Flow in member ${controller_index} according to \${json_body_add}.
     ${resp}    ClusterManagement.Post As Json To Member    ${ADD_BULK_CONFIG_NODES_API}    ${json_body_add}    ${controller_index}
 
+Add Table Flow
+    [Arguments]    ${json_body_add}    ${controller_index}
+    [Documentation]    Add Table in member ${controller_index} according to \${json_body_add}.
+    ${resp}    ClusterManagement.Post As Json To Member    ${ADD_TABLE_NODEs_API}    ${json_body_add}    ${controller_index}
+
 Delete Bulk Flow
     [Arguments]    ${json_body_del}    ${controller_index}
     [Documentation]    Delete Bulk Flow in member ${controller_index} according to \${json_body_del}.
@@ -68,6 +74,12 @@ Add Bulk Flow In Node
     Add Bulk Flow    ${json_body_add}    ${controller_index}
     Wait Until Write Finishes    ${controller_index}    ${timeout}
 
+Add Table In Node
+    [Arguments]    ${json_body_add}    ${controller_index}    ${timeout}
+    [Documentation]    Add Table Flow in member ${controller_index} and wait until operation is completed.
+    Add Table Flow    ${json_body_add}    ${controller_index}
+    Wait Until Write Finishes    ${controller_index}    ${timeout}
+
 Delete Bulk Flow In Node
     [Arguments]    ${json_body_del}    ${controller_index}    ${timeout}
     [Documentation]    Delete Bulk Flow in member ${controller_index} and wait until operation is completed.
index feec3dd57a7acf3529ac22a38c1d5250ba08aad8..c44d0470bcd8a5300b6d32ef6fd8ef0d2413746e 100644 (file)
@@ -135,6 +135,14 @@ Get_Raft_State_Of_Shard_At_Member
     [Arguments]    ${shard_name}    ${shard_type}    ${member_index}    ${verify_restconf}=False    ${http_timeout}=5
     [Documentation]    Send request to Jolokia on indexed member, return extracted Raft status.
     ...    Optionally, check restconf works.
+    ${raft_state} =    Get_Raft_Property_From_Shard_Member    RaftState    ${shard_name}    ${shard_type}    ${member_index}    verify_restconf=${verify_restconf}
+    ...    http_timeout=${http_timeout}
+    [Return]    ${raft_state}
+
+Get_Raft_Property_From_Shard_Member
+    [Arguments]    ${property}    ${shard_name}    ${shard_type}    ${member_index}    ${verify_restconf}=False    ${http_timeout}=5
+    [Documentation]    Send request to Jolokia on indexed member, return extracted Raft property.
+    ...    Optionally, check restconf works.
     ${session} =    Resolve_Http_Session_For_Member    member_index=${member_index}
     # TODO: Does the used URI tend to generate large data which floods log.html?
     BuiltIn.Run_Keyword_If    ${verify_restconf}    TemplatedRequests.Get_As_Json_Templated    session=${session}    folder=${RESTCONF_MODULES_DIR}    verify=False    http_timeout=${http_timeout}
@@ -143,8 +151,8 @@ Get_Raft_State_Of_Shard_At_Member
     ${data_text} =    TemplatedRequests.Get_As_Json_From_Uri    uri=${uri}    session=${session}    http_timeout=${http_timeout}
     ${data_object} =    RequestsLibrary.To_Json    ${data_text}
     ${value} =    Collections.Get_From_Dictionary    ${data_object}    value
-    ${raft_state} =    Collections.Get_From_Dictionary    ${value}    RaftState
-    [Return]    ${raft_state}
+    ${raft_property} =    Collections.Get_From_Dictionary    ${value}    ${property}
+    [Return]    ${raft_property}
 
 Verify_Shard_Leader_Elected
     [Arguments]    ${shard_name}    ${shard_type}    ${new_elected}    ${old_leader}    ${member_index_list}=${EMPTY}    ${verify_restconf}=True
diff --git a/csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot b/csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot
new file mode 100644 (file)
index 0000000..2a905c9
--- /dev/null
@@ -0,0 +1,147 @@
+*** Settings ***
+Documentation     Test suite for OF-18 Spurious Leader Election verification of term change while flows are added/deleted    #
+Suite Setup       ClusterManagement Setup
+Suite Teardown    Delete All Sessions
+Resource          ../../../libraries/BulkomaticKeywords.robot
+Resource          ../../../libraries/MininetKeywords.robot
+Resource          ../../../libraries/ClusterManagement.robot
+Resource          ../../../libraries/ClusterOpenFlow.robot
+Resource          ../../../libraries/Utils.robot
+Variables         ../../../variables/Variables.py
+
+*** Variables ***
+${operation_timeout}    400s
+${restart_timeout}    450s
+${flow_count_per_switch}    10000
+${flow_count_per_switch_ten_percent}    1000
+${switch_count}    1
+${flow_count_after_add}    10000
+${flow_count_after_del}    0
+${flow_count_after_del_ten_percent}    9000
+${orig_json_config_add}    sal_add_bulk_flow_config.json
+${orig_json_config_get}    sal_get_bulk_flow_config.json
+${orig_json_config_del}    sal_del_bulk_flow_config.json
+${orig_json_config_table_add}    add_table.json
+${shard_name}     inventory
+${shard_type}     config
+${verify_restconf}    False
+
+*** Test Cases ***
+Check Shard And Get Inventory
+    [Documentation]    Check Status for all shards and Finding a leader and followers in the inventory config shard
+    Check Shards Status And Initialize Variables
+    Get Inventory Follower
+
+Initial Current Term Verification
+    [Documentation]    Verifying current term for Leader Node Before
+    ${current_term_value_before}    Get_Current_Term_Of_Shard_At_Member
+    BuiltIn.Log to console    Current Term is ${current_term_value_before}
+    BuiltIn.Set Suite Variable    ${current_term_value_before}
+
+Add Bulk Flow From Follower
+    [Documentation]    ${flow_count_after_add} Flows added via Follower Node1 and verify it gets applied in all instances.
+    BulkomaticKeywords.Add Bulk Flow In Node    ${temp_json_config_add}    ${Follower_Node_1}    ${operation_timeout}
+
+Get Bulk Flows And Verify In Cluster
+    [Documentation]    Initiate get operation and check flow count across cluster nodes
+    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}    ${Inventory_Leader_List}
+
+Current Term Verification After Adding Bulk Flow
+    [Documentation]    Verifying current term for Leader Node after pushing the flows
+    ${current_term_value_after}    Get_Current_Term_Of_Shard_At_Member
+    BuiltIn.Set Suite Variable    ${current_term_value_after}
+
+Current Term Comparison Before And After Addition Of Flow
+    [Documentation]    Comparison of Current Term Before and After Addition of Flows
+    BuiltIn.Log to console    Current Term after pushing the flows is ${current_term_value_after}
+    Run Keyword If    ${current_term_value_before} == ${current_term_value_after}    Log    SUCCESS
+    ...    ELSE    Log    FAILURE
+    Should Be Equal    ${current_term_value_before}    ${current_term_value_after}
+
+Delete and Add ten percent of the flows for 5 iterations
+    [Documentation]    Performeing 5 iterations for Delete and Add ten Percentage of the flows
+    : FOR    ${index}    IN RANGE    1    6
+    \    Log    ${index}
+    \    BulkomaticKeywords.Delete Bulk Flow In Node    ${temp_json_config_del_ten_percent}    ${Follower_Node_1}    ${operation_timeout}
+    \    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_del_ten_percent}    ${Inventory_Leader_List}
+    \    BulkomaticKeywords.Add Bulk Flow In Node    ${temp_json_config_add_ten_percent}    ${Follower_Node_1}    ${operation_timeout}
+    \    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}    ${Inventory_Leader_List}
+
+Current Term Verification After Continuous Deletion and Addition Of Flows for 5 iterations
+    [Documentation]    Verifying current term for Leader Node after continuous deletion and addition of ten percent of the flows
+    ${current_term_value_after}    Get_Current_Term_Of_Shard_At_Member
+    BuiltIn.Set Suite Variable    ${current_term_value_after}
+
+Current Term Comparison Before and After Continuous Deletion and Addition Of Flows for 5 iterations
+    [Documentation]    Comparison Current Term Before and After Continuous Deletion and Addition Of Flows for 5 iterations
+    BuiltIn.Log to console    Current Term after pushing the flows is ${current_term_value_after}
+    Run Keyword If    ${current_term_value_before} == ${current_term_value_after}    Log    SUCCESS
+    ...    ELSE    Log    FAILURE
+    Should Be Equal    ${current_term_value_before}    ${current_term_value_after}
+
+Delete All Flows From Follower Node
+    [Documentation]    ${flow_count_after_add} Flows deleted via Leader Node and verify it gets applied in all instances.
+    BulkomaticKeywords.Delete Bulk Flow In Node    ${temp_json_config_del}    ${Follower_Node_1}    ${operation_timeout}
+
+Verify No Flows In Cluster After Flow Deletion
+    [Documentation]    Verifying No FLows in Cluster after Flows Deletion
+    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_del}    ${Inventory_Leader_List}
+
+PreLeader Verification
+    [Documentation]    Verifying LastIndex and LastApplied and compare both are equal
+    ${LastIndex}    Get_Last_Index_Of_Shard_At_Member
+    ${LastApplied}    Get_Last_Applied_Of_Shard_At_Member
+    Should Be Equal    ${LastIndex}    ${LastApplied}
+    BulkomaticKeywords.Add Table In Node    ${temp_json_config_table}    ${Follower_Node_1}    ${operation_timeout}
+    ${LastIndex}    Get_Last_Index_Of_Shard_At_Member
+    ${LastApplied}    Get_Last_Applied_Of_Shard_At_Member
+    Should Be Equal    ${LastIndex}    ${LastApplied}
+
+*** Keywords ***
+Check Shards Status And Initialize Variables
+    [Documentation]    Check Status for all shards in OpenFlow application.
+    ClusterOpenFlow.Check OpenFlow Shards Status
+    ${temp_json_config_add}    BulkomaticKeywords.Set DPN And Flow Count In Json Add    ${orig_json_config_add}    ${switch_count}    ${flow_count_per_switch}
+    ${temp_json_config_get}    BulkomaticKeywords.Set DPN And Flow Count In Json Get    ${orig_json_config_get}    ${switch_count}    ${flow_count_after_add}
+    ${temp_json_config_del}    BulkomaticKeywords.Set DPN And Flow Count In Json Del    ${orig_json_config_del}    ${switch_count}    ${flow_count_per_switch}
+    ${temp_json_config_del_ten_percent}    BulkomaticKeywords.Set DPN And Flow Count In Json Del    ${orig_json_config_del}    ${switch_count}    ${flow_count_per_switch_ten_percent}
+    ${temp_json_config_add_ten_percent}    BulkomaticKeywords.Set DPN And Flow Count In Json Add    ${orig_json_config_add}    ${switch_count}    ${flow_count_per_switch_ten_percent}
+    ${temp_json_config_table}    BulkomaticKeywords.Set DPN And Flow Count In Json Add    ${orig_json_config_table_add}    ${switch_count}    ${flow_count_per_switch}
+    Set Suite Variable    ${temp_json_config_add}
+    Set Suite Variable    ${temp_json_config_get}
+    Set Suite Variable    ${temp_json_config_del}
+    Set Suite Variable    ${temp_json_config_del_ten_percent}
+    Set Suite Variable    ${temp_json_config_add_ten_percent}
+    Set Suite Variable    ${temp_json_config_table}
+
+Get Inventory Follower
+    [Documentation]    Find a leader and followers in the inventory config shard
+    ${inventory_leader}    ${inventory_followers}    ClusterOpenFlow.Get InventoryConfig Shard Status
+    ${Follower_Node_1}=    Get From List    ${Inventory_Followers}    0
+    ${Follower_Node_2}=    Get From List    ${Inventory_Followers}    1
+    ${Inventory_Leader_List}=    Create List    ${inventory_leader}
+    ${Inventory_Follower_Node1_List}=    Create List    ${Follower_Node_1}
+    BuiltIn.Log to console    ${\n}
+    BuiltIn.Log to console    The follower Node1 is ${Follower_Node_1}
+    BuiltIn.Log to console    The follower Node2 is ${Follower_Node_2}
+    BuiltIn.Log to console    The leader Node is ${Inventory_Leader}
+    BuiltIn.Set Suite Variable    ${Follower_Node_1}
+    BuiltIn.Set Suite Variable    ${Follower_Node_2}
+    BuiltIn.Set Suite Variable    ${Inventory_Leader}
+    BuiltIn.Set Suite Variable    ${Inventory_Leader_List}
+    BuiltIn.Set Suite Variable    ${Inventory_Follower_Node1_List}
+
+Get_Current_Term_Of_Shard_At_Member
+    [Documentation]    Find a Raft Property Values From Shard Member
+    ${current_term_value} =    ClusterManagement.Get_Raft_Property_From_Shard_Member    CurrentTerm    ${shard_name}    ${shard_type}    ${Inventory_Leader}    ${verify_restconf}
+    [Return]    ${current_term_value}
+
+Get_Last_Index_Of_Shard_At_Member
+    [Documentation]    Find a leader and followers in the inventory config shard
+    ${LastIndex} =    ClusterManagement.Get_Raft_Property_From_Shard_Member    LastIndex    ${shard_name}    ${shard_type}    ${Inventory_Leader}    ${verify_restconf}
+    [Return]    ${LastIndex}
+
+Get_Last_Applied_Of_Shard_At_Member
+    [Documentation]    Find a leader and followers in the inventory config shard
+    ${LastApplied} =    ClusterManagement.Get_Raft_Property_From_Shard_Member    LastApplied    ${shard_name}    ${shard_type}    ${Inventory_Leader}    ${verify_restconf}
+    [Return]    ${LastApplied}
diff --git a/csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt b/csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt
new file mode 100644 (file)
index 0000000..9d96726
--- /dev/null
@@ -0,0 +1,4 @@
+# Place the suites in run order:
+integration/test/csit/suites/openflowplugin/Clustering_Bulkomatic/010__Cluster_Reconcilliation_Multi_DPN.robot
+integration/test/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot
+integration/test/csit/suites/openflowplugin/Clustering_Bulkomatic/030__Cluster_HA_Data_Recovery_BulkFlow_Single_Switch.robot
diff --git a/csit/variables/openflowplugin/add_table.json b/csit/variables/openflowplugin/add_table.json
new file mode 100644 (file)
index 0000000..9896fdb
--- /dev/null
@@ -0,0 +1,9 @@
+{
+  "input" :
+    {
+      "sal-bulk-flow:operation" : "add",
+      "sal-bulk-flow:dpn-count" : "2",
+      "sal-bulk-flow:start-table-id" : "0",
+      "sal-bulk-flow:end-table-id" : "9"
+    }
+}
diff --git a/csit/variables/openflowplugin/sal_table_add.json b/csit/variables/openflowplugin/sal_table_add.json
new file mode 100644 (file)
index 0000000..e9e4992
--- /dev/null
@@ -0,0 +1,16 @@
+{
+"input" :
+  {
+   "sal-bulk-flow:create-parents" : "false",
+   "sal-bulk-flow:is-add" : "true",
+   "sal-bulk-flow:dpn-count" : "1",
+   "sal-bulk-flow:flows-per-dpn" : "1",
+   "sal-bulk-flow:batch-size" : "1",
+   "sal-bulk-flow:seq" : "true",
+   "sal-bulk-flow:tx-chain" : "false",
+   "sal-bulk-flow:sleep-for" : "0",
+   "sal-bulk-flow:sleep-after" : "1",
+   "sal-bulk-flow:start-table-id" : "0",
+   "sal-bulk-flow:end-table-id" : "1"
+  }
+}
diff --git a/csit/variables/openflowplugin/sal_table_add_config.json b/csit/variables/openflowplugin/sal_table_add_config.json
new file mode 100644 (file)
index 0000000..f50f64c
--- /dev/null
@@ -0,0 +1,9 @@
+{
+  "input" :
+    {
+      "sal-bulk-flow:add" : "true",
+      "sal-bulk-flow:dpn-count" : "2",
+      "sal-bulk-flow:start-table-id" : "0",
+      "sal-bulk-flow:end-table-id" : "9"
+    }
+}
diff --git a/csit/variables/openflowplugin/sal_table_flow_config.json b/csit/variables/openflowplugin/sal_table_flow_config.json
new file mode 100644 (file)
index 0000000..e9e4992
--- /dev/null
@@ -0,0 +1,16 @@
+{
+"input" :
+  {
+   "sal-bulk-flow:create-parents" : "false",
+   "sal-bulk-flow:is-add" : "true",
+   "sal-bulk-flow:dpn-count" : "1",
+   "sal-bulk-flow:flows-per-dpn" : "1",
+   "sal-bulk-flow:batch-size" : "1",
+   "sal-bulk-flow:seq" : "true",
+   "sal-bulk-flow:tx-chain" : "false",
+   "sal-bulk-flow:sleep-for" : "0",
+   "sal-bulk-flow:sleep-after" : "1",
+   "sal-bulk-flow:start-table-id" : "0",
+   "sal-bulk-flow:end-table-id" : "1"
+  }
+}