From 6dcd6ab8611b4bad538a51b4f4d1048fd303f7d6 Mon Sep 17 00:00:00 2001 From: naveen Date: Tue, 11 Apr 2017 02:03:09 +0530 Subject: [PATCH] Verification of OF18 current Term Change-Id: I21cb25420d984db1da4f2a0d1e9a34c589afa3ce Signed-off-by: naveen --- csit/libraries/BulkomaticKeywords.robot | 12 ++ csit/libraries/ClusterManagement.robot | 12 +- ...rent_Term_Verification_3Node_Cluster.robot | 147 ++++++++++++++++++ ...ugin-bulkomatic-clustering-daily-boron.txt | 4 + csit/variables/openflowplugin/add_table.json | 9 ++ .../openflowplugin/sal_table_add.json | 16 ++ .../openflowplugin/sal_table_add_config.json | 9 ++ .../openflowplugin/sal_table_flow_config.json | 16 ++ 8 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot create mode 100644 csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt create mode 100644 csit/variables/openflowplugin/add_table.json create mode 100644 csit/variables/openflowplugin/sal_table_add.json create mode 100644 csit/variables/openflowplugin/sal_table_add_config.json create mode 100644 csit/variables/openflowplugin/sal_table_flow_config.json diff --git a/csit/libraries/BulkomaticKeywords.robot b/csit/libraries/BulkomaticKeywords.robot index c76bc7e75a..84e3697112 100644 --- a/csit/libraries/BulkomaticKeywords.robot +++ b/csit/libraries/BulkomaticKeywords.robot @@ -8,6 +8,7 @@ Variables ../variables/Variables.py *** Variables *** ${ADD_BULK_CONFIG_NODES_API} /restconf/operations/sal-bulk-flow:flow-test ${GET_BULK_CONFIG_NODES_API} /restconf/operations/sal-bulk-flow:read-flow-test +${ADD_TABLE_NODEs_API} /restconf/operations/sal-bulk-flow:table-test ${jolokia_write_op_status} /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/WriteOpStatus ${jolokia_read_op_status} /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/ReadOpStatus ${jolokia_flow_count_status} /jolokia/read/org.opendaylight.openflowplugin.applications.bulk.o.matic:type=FlowCounter/FlowCount @@ -38,6 +39,11 @@ Add Bulk Flow [Documentation] Add Bulk Flow in member ${controller_index} according to \${json_body_add}. ${resp} ClusterManagement.Post As Json To Member ${ADD_BULK_CONFIG_NODES_API} ${json_body_add} ${controller_index} +Add Table Flow + [Arguments] ${json_body_add} ${controller_index} + [Documentation] Add Table in member ${controller_index} according to \${json_body_add}. + ${resp} ClusterManagement.Post As Json To Member ${ADD_TABLE_NODEs_API} ${json_body_add} ${controller_index} + Delete Bulk Flow [Arguments] ${json_body_del} ${controller_index} [Documentation] Delete Bulk Flow in member ${controller_index} according to \${json_body_del}. @@ -68,6 +74,12 @@ Add Bulk Flow In Node Add Bulk Flow ${json_body_add} ${controller_index} Wait Until Write Finishes ${controller_index} ${timeout} +Add Table In Node + [Arguments] ${json_body_add} ${controller_index} ${timeout} + [Documentation] Add Table Flow in member ${controller_index} and wait until operation is completed. + Add Table Flow ${json_body_add} ${controller_index} + Wait Until Write Finishes ${controller_index} ${timeout} + Delete Bulk Flow In Node [Arguments] ${json_body_del} ${controller_index} ${timeout} [Documentation] Delete Bulk Flow in member ${controller_index} and wait until operation is completed. diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot index feec3dd57a..c44d0470bc 100644 --- a/csit/libraries/ClusterManagement.robot +++ b/csit/libraries/ClusterManagement.robot @@ -135,6 +135,14 @@ Get_Raft_State_Of_Shard_At_Member [Arguments] ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False ${http_timeout}=5 [Documentation] Send request to Jolokia on indexed member, return extracted Raft status. ... Optionally, check restconf works. + ${raft_state} = Get_Raft_Property_From_Shard_Member RaftState ${shard_name} ${shard_type} ${member_index} verify_restconf=${verify_restconf} + ... http_timeout=${http_timeout} + [Return] ${raft_state} + +Get_Raft_Property_From_Shard_Member + [Arguments] ${property} ${shard_name} ${shard_type} ${member_index} ${verify_restconf}=False ${http_timeout}=5 + [Documentation] Send request to Jolokia on indexed member, return extracted Raft property. + ... Optionally, check restconf works. ${session} = Resolve_Http_Session_For_Member member_index=${member_index} # TODO: Does the used URI tend to generate large data which floods log.html? BuiltIn.Run_Keyword_If ${verify_restconf} TemplatedRequests.Get_As_Json_Templated session=${session} folder=${RESTCONF_MODULES_DIR} verify=False http_timeout=${http_timeout} @@ -143,8 +151,8 @@ Get_Raft_State_Of_Shard_At_Member ${data_text} = TemplatedRequests.Get_As_Json_From_Uri uri=${uri} session=${session} http_timeout=${http_timeout} ${data_object} = RequestsLibrary.To_Json ${data_text} ${value} = Collections.Get_From_Dictionary ${data_object} value - ${raft_state} = Collections.Get_From_Dictionary ${value} RaftState - [Return] ${raft_state} + ${raft_property} = Collections.Get_From_Dictionary ${value} ${property} + [Return] ${raft_property} Verify_Shard_Leader_Elected [Arguments] ${shard_name} ${shard_type} ${new_elected} ${old_leader} ${member_index_list}=${EMPTY} ${verify_restconf}=True diff --git a/csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot b/csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot new file mode 100644 index 0000000000..2a905c921d --- /dev/null +++ b/csit/suites/openflowplugin/Clustering_Bulkomatic/040__Cluster_Current_Term_Verification_3Node_Cluster.robot @@ -0,0 +1,147 @@ +*** Settings *** +Documentation Test suite for OF-18 Spurious Leader Election verification of term change while flows are added/deleted # +Suite Setup ClusterManagement Setup +Suite Teardown Delete All Sessions +Resource ../../../libraries/BulkomaticKeywords.robot +Resource ../../../libraries/MininetKeywords.robot +Resource ../../../libraries/ClusterManagement.robot +Resource ../../../libraries/ClusterOpenFlow.robot +Resource ../../../libraries/Utils.robot +Variables ../../../variables/Variables.py + +*** Variables *** +${operation_timeout} 400s +${restart_timeout} 450s +${flow_count_per_switch} 10000 +${flow_count_per_switch_ten_percent} 1000 +${switch_count} 1 +${flow_count_after_add} 10000 +${flow_count_after_del} 0 +${flow_count_after_del_ten_percent} 9000 +${orig_json_config_add} sal_add_bulk_flow_config.json +${orig_json_config_get} sal_get_bulk_flow_config.json +${orig_json_config_del} sal_del_bulk_flow_config.json +${orig_json_config_table_add} add_table.json +${shard_name} inventory +${shard_type} config +${verify_restconf} False + +*** Test Cases *** +Check Shard And Get Inventory + [Documentation] Check Status for all shards and Finding a leader and followers in the inventory config shard + Check Shards Status And Initialize Variables + Get Inventory Follower + +Initial Current Term Verification + [Documentation] Verifying current term for Leader Node Before + ${current_term_value_before} Get_Current_Term_Of_Shard_At_Member + BuiltIn.Log to console Current Term is ${current_term_value_before} + BuiltIn.Set Suite Variable ${current_term_value_before} + +Add Bulk Flow From Follower + [Documentation] ${flow_count_after_add} Flows added via Follower Node1 and verify it gets applied in all instances. + BulkomaticKeywords.Add Bulk Flow In Node ${temp_json_config_add} ${Follower_Node_1} ${operation_timeout} + +Get Bulk Flows And Verify In Cluster + [Documentation] Initiate get operation and check flow count across cluster nodes + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} ${Inventory_Leader_List} + +Current Term Verification After Adding Bulk Flow + [Documentation] Verifying current term for Leader Node after pushing the flows + ${current_term_value_after} Get_Current_Term_Of_Shard_At_Member + BuiltIn.Set Suite Variable ${current_term_value_after} + +Current Term Comparison Before And After Addition Of Flow + [Documentation] Comparison of Current Term Before and After Addition of Flows + BuiltIn.Log to console Current Term after pushing the flows is ${current_term_value_after} + Run Keyword If ${current_term_value_before} == ${current_term_value_after} Log SUCCESS + ... ELSE Log FAILURE + Should Be Equal ${current_term_value_before} ${current_term_value_after} + +Delete and Add ten percent of the flows for 5 iterations + [Documentation] Performeing 5 iterations for Delete and Add ten Percentage of the flows + : FOR ${index} IN RANGE 1 6 + \ Log ${index} + \ BulkomaticKeywords.Delete Bulk Flow In Node ${temp_json_config_del_ten_percent} ${Follower_Node_1} ${operation_timeout} + \ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del_ten_percent} ${Inventory_Leader_List} + \ BulkomaticKeywords.Add Bulk Flow In Node ${temp_json_config_add_ten_percent} ${Follower_Node_1} ${operation_timeout} + \ BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} ${Inventory_Leader_List} + +Current Term Verification After Continuous Deletion and Addition Of Flows for 5 iterations + [Documentation] Verifying current term for Leader Node after continuous deletion and addition of ten percent of the flows + ${current_term_value_after} Get_Current_Term_Of_Shard_At_Member + BuiltIn.Set Suite Variable ${current_term_value_after} + +Current Term Comparison Before and After Continuous Deletion and Addition Of Flows for 5 iterations + [Documentation] Comparison Current Term Before and After Continuous Deletion and Addition Of Flows for 5 iterations + BuiltIn.Log to console Current Term after pushing the flows is ${current_term_value_after} + Run Keyword If ${current_term_value_before} == ${current_term_value_after} Log SUCCESS + ... ELSE Log FAILURE + Should Be Equal ${current_term_value_before} ${current_term_value_after} + +Delete All Flows From Follower Node + [Documentation] ${flow_count_after_add} Flows deleted via Leader Node and verify it gets applied in all instances. + BulkomaticKeywords.Delete Bulk Flow In Node ${temp_json_config_del} ${Follower_Node_1} ${operation_timeout} + +Verify No Flows In Cluster After Flow Deletion + [Documentation] Verifying No FLows in Cluster after Flows Deletion + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} ${Inventory_Leader_List} + +PreLeader Verification + [Documentation] Verifying LastIndex and LastApplied and compare both are equal + ${LastIndex} Get_Last_Index_Of_Shard_At_Member + ${LastApplied} Get_Last_Applied_Of_Shard_At_Member + Should Be Equal ${LastIndex} ${LastApplied} + BulkomaticKeywords.Add Table In Node ${temp_json_config_table} ${Follower_Node_1} ${operation_timeout} + ${LastIndex} Get_Last_Index_Of_Shard_At_Member + ${LastApplied} Get_Last_Applied_Of_Shard_At_Member + Should Be Equal ${LastIndex} ${LastApplied} + +*** Keywords *** +Check Shards Status And Initialize Variables + [Documentation] Check Status for all shards in OpenFlow application. + ClusterOpenFlow.Check OpenFlow Shards Status + ${temp_json_config_add} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_add} ${switch_count} ${flow_count_per_switch} + ${temp_json_config_get} BulkomaticKeywords.Set DPN And Flow Count In Json Get ${orig_json_config_get} ${switch_count} ${flow_count_after_add} + ${temp_json_config_del} BulkomaticKeywords.Set DPN And Flow Count In Json Del ${orig_json_config_del} ${switch_count} ${flow_count_per_switch} + ${temp_json_config_del_ten_percent} BulkomaticKeywords.Set DPN And Flow Count In Json Del ${orig_json_config_del} ${switch_count} ${flow_count_per_switch_ten_percent} + ${temp_json_config_add_ten_percent} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_add} ${switch_count} ${flow_count_per_switch_ten_percent} + ${temp_json_config_table} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_table_add} ${switch_count} ${flow_count_per_switch} + Set Suite Variable ${temp_json_config_add} + Set Suite Variable ${temp_json_config_get} + Set Suite Variable ${temp_json_config_del} + Set Suite Variable ${temp_json_config_del_ten_percent} + Set Suite Variable ${temp_json_config_add_ten_percent} + Set Suite Variable ${temp_json_config_table} + +Get Inventory Follower + [Documentation] Find a leader and followers in the inventory config shard + ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status + ${Follower_Node_1}= Get From List ${Inventory_Followers} 0 + ${Follower_Node_2}= Get From List ${Inventory_Followers} 1 + ${Inventory_Leader_List}= Create List ${inventory_leader} + ${Inventory_Follower_Node1_List}= Create List ${Follower_Node_1} + BuiltIn.Log to console ${\n} + BuiltIn.Log to console The follower Node1 is ${Follower_Node_1} + BuiltIn.Log to console The follower Node2 is ${Follower_Node_2} + BuiltIn.Log to console The leader Node is ${Inventory_Leader} + BuiltIn.Set Suite Variable ${Follower_Node_1} + BuiltIn.Set Suite Variable ${Follower_Node_2} + BuiltIn.Set Suite Variable ${Inventory_Leader} + BuiltIn.Set Suite Variable ${Inventory_Leader_List} + BuiltIn.Set Suite Variable ${Inventory_Follower_Node1_List} + +Get_Current_Term_Of_Shard_At_Member + [Documentation] Find a Raft Property Values From Shard Member + ${current_term_value} = ClusterManagement.Get_Raft_Property_From_Shard_Member CurrentTerm ${shard_name} ${shard_type} ${Inventory_Leader} ${verify_restconf} + [Return] ${current_term_value} + +Get_Last_Index_Of_Shard_At_Member + [Documentation] Find a leader and followers in the inventory config shard + ${LastIndex} = ClusterManagement.Get_Raft_Property_From_Shard_Member LastIndex ${shard_name} ${shard_type} ${Inventory_Leader} ${verify_restconf} + [Return] ${LastIndex} + +Get_Last_Applied_Of_Shard_At_Member + [Documentation] Find a leader and followers in the inventory config shard + ${LastApplied} = ClusterManagement.Get_Raft_Property_From_Shard_Member LastApplied ${shard_name} ${shard_type} ${Inventory_Leader} ${verify_restconf} + [Return] ${LastApplied} diff --git a/csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt b/csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt new file mode 100644 index 0000000000..9d96726f05 --- /dev/null +++ b/csit/testplans/openflowplugin-bulkomatic-clustering-daily-boron.txt @@ -0,0 +1,4 @@ +# Place the suites in run order: +integration/test/csit/suites/openflowplugin/Clustering_Bulkomatic/010__Cluster_Reconcilliation_Multi_DPN.robot +integration/test/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot +integration/test/csit/suites/openflowplugin/Clustering_Bulkomatic/030__Cluster_HA_Data_Recovery_BulkFlow_Single_Switch.robot diff --git a/csit/variables/openflowplugin/add_table.json b/csit/variables/openflowplugin/add_table.json new file mode 100644 index 0000000000..9896fdbf98 --- /dev/null +++ b/csit/variables/openflowplugin/add_table.json @@ -0,0 +1,9 @@ +{ + "input" : + { + "sal-bulk-flow:operation" : "add", + "sal-bulk-flow:dpn-count" : "2", + "sal-bulk-flow:start-table-id" : "0", + "sal-bulk-flow:end-table-id" : "9" + } +} diff --git a/csit/variables/openflowplugin/sal_table_add.json b/csit/variables/openflowplugin/sal_table_add.json new file mode 100644 index 0000000000..e9e4992ac8 --- /dev/null +++ b/csit/variables/openflowplugin/sal_table_add.json @@ -0,0 +1,16 @@ +{ +"input" : + { + "sal-bulk-flow:create-parents" : "false", + "sal-bulk-flow:is-add" : "true", + "sal-bulk-flow:dpn-count" : "1", + "sal-bulk-flow:flows-per-dpn" : "1", + "sal-bulk-flow:batch-size" : "1", + "sal-bulk-flow:seq" : "true", + "sal-bulk-flow:tx-chain" : "false", + "sal-bulk-flow:sleep-for" : "0", + "sal-bulk-flow:sleep-after" : "1", + "sal-bulk-flow:start-table-id" : "0", + "sal-bulk-flow:end-table-id" : "1" + } +} diff --git a/csit/variables/openflowplugin/sal_table_add_config.json b/csit/variables/openflowplugin/sal_table_add_config.json new file mode 100644 index 0000000000..f50f64cee9 --- /dev/null +++ b/csit/variables/openflowplugin/sal_table_add_config.json @@ -0,0 +1,9 @@ +{ + "input" : + { + "sal-bulk-flow:add" : "true", + "sal-bulk-flow:dpn-count" : "2", + "sal-bulk-flow:start-table-id" : "0", + "sal-bulk-flow:end-table-id" : "9" + } +} diff --git a/csit/variables/openflowplugin/sal_table_flow_config.json b/csit/variables/openflowplugin/sal_table_flow_config.json new file mode 100644 index 0000000000..e9e4992ac8 --- /dev/null +++ b/csit/variables/openflowplugin/sal_table_flow_config.json @@ -0,0 +1,16 @@ +{ +"input" : + { + "sal-bulk-flow:create-parents" : "false", + "sal-bulk-flow:is-add" : "true", + "sal-bulk-flow:dpn-count" : "1", + "sal-bulk-flow:flows-per-dpn" : "1", + "sal-bulk-flow:batch-size" : "1", + "sal-bulk-flow:seq" : "true", + "sal-bulk-flow:tx-chain" : "false", + "sal-bulk-flow:sleep-for" : "0", + "sal-bulk-flow:sleep-after" : "1", + "sal-bulk-flow:start-table-id" : "0", + "sal-bulk-flow:end-table-id" : "1" + } +} -- 2.36.6