From: Sanjib Mohapatra Date: Wed, 15 Jun 2016 09:24:59 +0000 (+0530) Subject: Reconcilliation in a 3 node cluster multi DPN environment X-Git-Tag: release/beryllium-sr3~85 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=commitdiff_plain;h=8c580a382022c115dc7a572d0134c1af8db1501e;p=integration%2Ftest.git Reconcilliation in a 3 node cluster multi DPN environment Description: In a 3 node cluster initial inventory shard status is verified and following tests are performed - Start Mininet 3 DPNs connect to one of the follower, add bulk 3K flows via that node (1K per DPN). - Verify flows in config DS as well as in switch, Stop Mininet and reconnect DPNs to same follower node and verify same flows installed in respective DPNs. - Stop Mininet and connect back to 2nd follower node and verify same flows installed in respective DPNs. - Stop Mininet and connect back to leader node and verify same flows installed in respective DPNs. - Also modified existing BulkomaticKeywords.robot library for implementing KWs to get json object instead of json files as argument. - Corresponding changes are done in rest of the Clustering_Bulkomatic test suites. Change-Id: Ife65746c13681377f63e152bbf88392e492924d2 Signed-off-by: Sanjib Mohapatra --- diff --git a/csit/libraries/BulkomaticKeywords.robot b/csit/libraries/BulkomaticKeywords.robot index e6b27fb426..a2fae8455a 100644 --- a/csit/libraries/BulkomaticKeywords.robot +++ b/csit/libraries/BulkomaticKeywords.robot @@ -35,22 +35,19 @@ Wait Until Read Finishes Wait Until Keyword Succeeds ${timeout} 1s BulkomaticKeywords.Operation Status Check ${controller_index} ${jolokia_read_op_status} Add Bulk Flow - [Arguments] ${controller_index} ${add_bulk_json_file} - [Documentation] Add Bulk Flow in ${controller_index} according to ${add_bulk_json_file}. - ${add_body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${add_bulk_json_file} - ${resp} Utils.Post Elements To URI ${ADD_BULK_CONFIG_NODES_API} ${add_body} headers=${HEADERS_YANG_JSON} session=controller${controller_index} + [Arguments] ${controller_index} ${json_body_add} + [Documentation] Add Bulk Flow in ${controller_index} according to ${json_body_add}. + ${resp} Utils.Post Elements To URI ${ADD_BULK_CONFIG_NODES_API} ${json_body_add} headers=${HEADERS_YANG_JSON} session=controller${controller_index} Delete Bulk Flow - [Arguments] ${controller_index} ${del_bulk_json_file} - [Documentation] Delete Bulk Flow in ${controller_index} according to ${del_bulk_json_file}. - ${del_body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${del_bulk_json_file} - ${resp} Utils.Post Elements To URI ${ADD_BULK_CONFIG_NODES_API} ${del_body} headers=${HEADERS_YANG_JSON} session=controller${controller_index} + [Arguments] ${controller_index} ${json_body_del} + [Documentation] Delete Bulk Flow in ${controller_index} according to ${json_body_del}. + ${resp} Utils.Post Elements To URI ${ADD_BULK_CONFIG_NODES_API} ${json_body_del} headers=${HEADERS_YANG_JSON} session=controller${controller_index} Get Bulk Flow - [Arguments] ${controller_index} ${get_bulk_json_file} - [Documentation] Get Bulk Flow in ${controller_index} according to ${get_bulk_json_file}. - ${get_body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${get_bulk_json_file} - ${resp} Utils.Post Elements To URI ${GET_BULK_CONFIG_NODES_API} ${get_body} headers=${HEADERS_YANG_JSON} session=controller${controller_index} + [Arguments] ${controller_index} ${json_body_get} + [Documentation] Get Bulk Flow in ${controller_index} according to ${json_body_get}. + ${resp} Utils.Post Elements To URI ${GET_BULK_CONFIG_NODES_API} ${json_body_get} headers=${HEADERS_YANG_JSON} session=controller${controller_index} Get Bulk Flow Count [Arguments] ${controller_index} @@ -69,23 +66,62 @@ Verify Flow Count Should Be Equal As Strings ${value} ${flow_count} Add Bulk Flow In Node - [Arguments] ${controller_index} ${add_bulk_json_file} ${timeout} + [Arguments] ${controller_index} ${json_body_add} ${timeout} [Documentation] Add Bulk Flow in ${controller_index} and wait until operation is completed. - Add Bulk Flow ${controller_index} ${add_bulk_json_file} + Add Bulk Flow ${controller_index} ${json_body_add} Wait Until Write Finishes ${controller_index} ${timeout} Delete Bulk Flow In Node - [Arguments] ${controller_index} ${delete_bulk_json_file} ${timeout} + [Arguments] ${controller_index} ${json_body_del} ${timeout} [Documentation] Delete Bulk Flow in ${controller_index} and wait until operation is completed. - Delete Bulk Flow ${controller_index} ${delete_bulk_json_file} + Delete Bulk Flow ${controller_index} ${json_body_del} Wait Until Write Finishes ${controller_index} ${timeout} Get Bulk Flow And Verify Count In Cluster - [Arguments] ${controller_index_list} ${get_bulk_json_file} ${timeout} ${flow_count} + [Arguments] ${controller_index_list} ${json_body_get} ${timeout} ${flow_count} [Documentation] Get Bulk Flow and Verify Flow Count in ${controller_index_list} matches ${flow_count}. : FOR ${index} IN @{controller_index_list} - \ Get Bulk Flow ${index} ${get_bulk_json_file} + \ Get Bulk Flow ${index} ${json_body_get} : FOR ${index} IN @{controller_index_list} \ Wait Until Read Finishes ${index} ${timeout} : FOR ${index} IN @{controller_index_list} \ Verify Flow Count ${index} ${flow_count} + +Set DPN And Flow Count In Json Add + [Arguments] ${json_config} ${dpn_count} ${flows_count} + [Documentation] Set new DPN count and flows count per DPN in the Bulkomatic Add json file. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${json_config} + ${get_string}= Set Variable "sal-bulk-flow:dpn-count" : "1" + ${put_string}= Set Variable "sal-bulk-flow:dpn-count" : "${dpn_count}" + ${str} Replace String Using Regexp ${body} ${get_string} ${put_string} + ${get_string}= Set Variable "sal-bulk-flow:flows-per-dpn" : "1000" + ${put_string}= Set Variable "sal-bulk-flow:flows-per-dpn" : "${flows_count}" + ${json_body_add} Replace String Using Regexp ${str} ${get_string} ${put_string} + Log ${json_body_add} + [Return] ${json_body_add} + +Set DPN And Flow Count In Json Get + [Arguments] ${json_config} ${dpn_count} ${flows_count} + [Documentation] Set new DPN count and flows count per DPN in the Bulkomatic Get json file. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${json_config} + ${get_string}= Set Variable "sal-bulk-flow:dpn-count" : "1" + ${put_string}= Set Variable "sal-bulk-flow:dpn-count" : "${dpn_count}" + ${str} Replace String Using Regexp ${body} ${get_string} ${put_string} + ${get_string}= Set Variable "sal-bulk-flow:flows-per-dpn" : "1000" + ${put_string}= Set Variable "sal-bulk-flow:flows-per-dpn" : "${flows_count}" + ${json_body_get} Replace String Using Regexp ${str} ${get_string} ${put_string} + Log ${json_body_get} + [Return] ${json_body_get} + +Set DPN And Flow Count In Json Del + [Arguments] ${json_config} ${dpn_count} ${flows_count} + [Documentation] Set new DPN count and flows count per DPN in the Bulkomatic Del json file. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/${json_config} + ${get_string}= Set Variable "sal-bulk-flow:dpn-count" : "1" + ${put_string}= Set Variable "sal-bulk-flow:dpn-count" : "${dpn_count}" + ${str} Replace String Using Regexp ${body} ${get_string} ${put_string} + ${get_string}= Set Variable "sal-bulk-flow:flows-per-dpn" : "1000" + ${put_string}= Set Variable "sal-bulk-flow:flows-per-dpn" : "${flows_count}" + ${json_body_del} Replace String Using Regexp ${str} ${get_string} ${put_string} + Log ${json_body_del} + [Return] ${json_body_del} diff --git a/csit/suites/openflowplugin/Clustering_Bulkomatic/010__Cluster_HA_Data_Recovery_BulkFlow_Single_Switch.robot b/csit/suites/openflowplugin/Clustering_Bulkomatic/010__Cluster_HA_Data_Recovery_BulkFlow_Single_Switch.robot index 81fc8e327f..64aaca5aba 100644 --- a/csit/suites/openflowplugin/Clustering_Bulkomatic/010__Cluster_HA_Data_Recovery_BulkFlow_Single_Switch.robot +++ b/csit/suites/openflowplugin/Clustering_Bulkomatic/010__Cluster_HA_Data_Recovery_BulkFlow_Single_Switch.robot @@ -13,12 +13,12 @@ Variables ../../../variables/Variables.py ${operation_timeout} 100s ${restart_timeout} 350s ${flow_count_per_switch} 1000 -${switch_count_per_node} 1 +${switch_count} 1 ${flow_count_after_add} 1000 ${flow_count_after_del} 0 -${add_small_config} sal_add_bulk_flow_small_config.json -${get_small_config} sal_get_bulk_flow_small_config.json -${del_small_config} sal_del_bulk_flow_small_config.json +${orig_json_config_add} sal_add_bulk_flow_config.json +${orig_json_config_get} sal_get_bulk_flow_config.json +${orig_json_config_del} sal_del_bulk_flow_config.json *** Test Cases *** Create Original Cluster List @@ -43,14 +43,19 @@ Start Mininet Connect To Follower Node1 Add Bulk Flow From Follower [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${add_small_config} ${operation_timeout} + ${temp_json_config_add} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_add} ${switch_count} ${flow_count_per_switch} + Set Suite Variable ${temp_json_config_add} + BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_add} ${operation_timeout} Get Bulk Flows and Verify In Cluster - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_add} + [Documentation] Initiate get operation and check flow count across cluster nodes + ${temp_json_config_get} BulkomaticKeywords.Set DPN And Flow Count In Json Get ${orig_json_config_get} ${switch_count} ${flow_count_after_add} + Set Suite Variable ${temp_json_config_get} + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} Verify Flows In Switch Before Cluster Restart [Documentation] Verify flows are installed in switch before cluster restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Kill All Cluster Nodes [Documentation] Kill All Nodes. @@ -67,7 +72,7 @@ Restart All Cluster Nodes Verify Data Recovery After Cluster Restart [Documentation] 1000 Flows preserved in all controller instances. - Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} + Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ... ${flow_count_after_add} Start Mininet Again Connect To Follower Node1 @@ -77,7 +82,7 @@ Start Mininet Again Connect To Follower Node1 Verify Flows In Switch After Cluster Restart [Documentation] Verify flows are installed in switch after cluster restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Stop Mininet Connected To Follower Node1 [Documentation] Stop mininet and exit connection. @@ -85,11 +90,14 @@ Stop Mininet Connected To Follower Node1 Utils.Clean Mininet System Delete All Flows From Follower Node1 - [Documentation] Rest 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${del_small_config} ${operation_timeout} + [Documentation] 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances. + ${temp_json_config_del} BulkomaticKeywords.Set DPN And Flow Count In Json Del ${orig_json_config_del} ${switch_count} ${flow_count_per_switch} + Set Suite Variable ${temp_json_config_del} + BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_del} ${operation_timeout} Verify No Flows In Cluster - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del} + [Documentation] Verify flow count is 0 across cluster nodes. + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} Get Inventory Leader Before Leader Restart [Documentation] Find leader in the inventory config shard @@ -103,17 +111,18 @@ Start Mininet Connect To Leader Add Bulk Flow From Leader [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Add Bulk Flow In Node ${Inventory_Leader} ${add_small_config} ${operation_timeout} + BulkomaticKeywords.Add Bulk Flow In Node ${Inventory_Leader} ${temp_json_config_add} ${operation_timeout} Get Bulk Flows and Verify In Cluster Before Leader Restart - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_add} + [Documentation] Initiate get operation and check flow count across cluster nodes + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} Verify Flows In Switch Before Leader Restart [Documentation] Verify flows are installed in switch before leader restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Kill Leader From Cluster Node - [Documentation] Kill Leader Node and Start it Up. + [Documentation] Kill Leader Node. ClusterKeywords.Kill Multiple Controllers ${Inventory_Leader} Stop Mininet Connected To Leader Node @@ -127,7 +136,7 @@ Restart Leader from Cluster Node Verify Data Recovery After Leader Restart [Documentation] 1000 Flows preserved in all controller instances. - Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} + Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ... ${flow_count_after_add} Start Mininet Again Connect To Leader @@ -137,7 +146,7 @@ Start Mininet Again Connect To Leader Verify Flows In Switch After Leader Restart [Documentation] Verify flows are installed in switch after leader restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Stop Mininet Connected To Leader Node After Leader Restart [Documentation] Stop mininet and exit connection. @@ -145,11 +154,12 @@ Stop Mininet Connected To Leader Node After Leader Restart Utils.Clean Mininet System Delete All Flows From Leader Node - [Documentation] Rest 1000 Flows deleted via Leader Node and verify it gets applied in all instances. - BulkomaticKeywords.Delete Bulk Flow In Node ${Inventory_Leader} ${del_small_config} ${operation_timeout} + [Documentation] 1000 Flows deleted via Leader Node and verify it gets applied in all instances. + BulkomaticKeywords.Delete Bulk Flow In Node ${Inventory_Leader} ${temp_json_config_del} ${operation_timeout} Verify No Flows In Cluster After Leader Restart - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del} + [Documentation] Verify flow count is 0 across cluster nodes. + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} Get Inventory Follower Before follower Restart [Documentation] Find follower in the inventory config shard @@ -164,14 +174,15 @@ Start Mininet Connect To Follower Node2 Add Bulk Flow From Follower Node2 [Documentation] 1000 Flows added via Follower Node2 and verify it gets applied in all instances. - BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_2} ${add_small_config} ${operation_timeout} + BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_2} ${temp_json_config_add} ${operation_timeout} Get Bulk Flows and Verify In Cluster Before Follower Restart - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_add} + [Documentation] Initiate get operation and check flow count across cluster nodes + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} Verify Flows In Switch Before Follower Restart [Documentation] Verify flows are installed in switch before follower restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Kill Follower Node2 [Documentation] Kill Follower Node2. @@ -188,7 +199,7 @@ Restart Follower Node2 Verify Data Recovery After Follower Node2 Restart [Documentation] 1000 Flows preserved in all controller instances. - Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} + Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ... ${flow_count_after_add} Start Mininet Again Connect To Follower Node2 @@ -198,7 +209,7 @@ Start Mininet Again Connect To Follower Node2 Verify Flows In Switch After Follower Node2 Restart [Documentation] Verify flows are installed in switch after follower restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Stop Mininet Connected To Follower Node2 [Documentation] Stop mininet and exit connection. @@ -206,8 +217,8 @@ Stop Mininet Connected To Follower Node2 Utils.Clean Mininet System Delete All Flows From Follower Node 2 - [Documentation] Rest 1000 Flows deleted via Leader Node and verify it gets applied in all instances. - BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_2} ${del_small_config} ${operation_timeout} + [Documentation] 1000 Flows deleted via Leader Node and verify it gets applied in all instances. + BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_2} ${temp_json_config_del} ${operation_timeout} Verify No Flows In Cluster After Follower Node2 Restart - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del} + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} diff --git a/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot b/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot index f26a83031d..3ccd2e6ef8 100644 --- a/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot +++ b/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot @@ -13,12 +13,12 @@ Variables ../../../variables/Variables.py ${operation_timeout} 100s ${restart_timeout} 350s ${flow_count_per_switch} 1000 -${switch_count_per_node} 1 +${switch_count} 1 ${flow_count_after_add} 1000 ${flow_count_after_del} 0 -${add_small_config} sal_add_bulk_flow_small_config.json -${get_small_config} sal_get_bulk_flow_small_config.json -${del_small_config} sal_del_bulk_flow_small_config.json +${orig_json_config_add} sal_add_bulk_flow_config.json +${orig_json_config_get} sal_get_bulk_flow_config.json +${orig_json_config_del} sal_del_bulk_flow_config.json *** Test Cases *** Create Original Cluster List @@ -55,14 +55,19 @@ Start Mininet Connect To Follower Node1 Add Bulk Flow From Follower [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${add_small_config} ${operation_timeout} + ${temp_json_config_add} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_add} ${switch_count} ${flow_count_per_switch} + Set Suite Variable ${temp_json_config_add} + BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_add} ${operation_timeout} Get Bulk Flows and Verify In Cluster - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${Inventory_Followers} ${get_small_config} ${operation_timeout} ${flow_count_after_add} + [Documentation] Initiate get operation and check flow count across cluster nodes. + ${temp_json_config_get} BulkomaticKeywords.Set DPN And Flow Count In Json Get ${orig_json_config_get} ${switch_count} ${flow_count_after_add} + Set Suite Variable ${temp_json_config_get} + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${Inventory_Followers} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} Verify Flows In Switch Before Cluster Restart [Documentation] Verify flows are installed in switch before cluster restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Restart Leader From Cluster Node [Documentation] Restart Leader Node. @@ -70,12 +75,12 @@ Restart Leader From Cluster Node Verify Data Recovery After Leader Restart [Documentation] 1000 Flows preserved in all controller instances. - Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} + Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ... ${flow_count_after_add} Verify Flows In Switch After Leader Restart [Documentation] Verify flows are installed in switch after cluster restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Stop Mininet Connected To Follower Node1 [Documentation] Stop mininet and exit connection. @@ -83,11 +88,14 @@ Stop Mininet Connected To Follower Node1 Utils.Clean Mininet System Delete All Flows From Follower Node1 - [Documentation] Rest 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${del_small_config} ${operation_timeout} + [Documentation] 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances. + ${temp_json_config_del} BulkomaticKeywords.Set DPN And Flow Count In Json Del ${orig_json_config_del} ${switch_count} ${flow_count_per_switch} + Set Suite Variable ${temp_json_config_del} + BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_del} ${operation_timeout} Verify No Flows In Cluster - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del} + [Documentation] Verify flow count is 0 across cluster nodes. + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} Get Inventory Follower Before Follower Restart [Documentation] Find Leader and followers in the inventory config shard @@ -115,14 +123,15 @@ Start Mininet Connect To Follower Node Add Bulk Flow From Follower Node1 [Documentation] 1000 Flows added via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${add_small_config} ${operation_timeout} + BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_add} ${operation_timeout} Get Bulk Flows and Verify In Cluster Before Follower Restart - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${Active_Nodes} ${get_small_config} ${operation_timeout} ${flow_count_after_add} + [Documentation] Initiate get operation and check flow count only across active cluster nodes + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${Active_Nodes} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} Verify Flows In Switch Before Follower Restart [Documentation] Verify flows are installed in switch before follower restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Restart Follower From Cluster Node [Documentation] Restart Follower Node2. @@ -130,12 +139,12 @@ Restart Follower From Cluster Node Verify Data Recovery After Follower Restart [Documentation] 1000 Flows preserved in all controller instances. - Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} + Wait Until Keyword Succeeds ${restart_timeout} 2s BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ... ${flow_count_after_add} Verify Flows In Switch After Follower Restart [Documentation] Verify flows are installed in switch after cluster restart. - MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count_per_node} ${flow_count_per_switch} ${operation_timeout} + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} Stop Mininet Connected To Follower Node [Documentation] Stop mininet and exit connection. @@ -143,8 +152,9 @@ Stop Mininet Connected To Follower Node Utils.Clean Mininet System Delete All Flows From Follower Node - [Documentation] Rest 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances. - BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${del_small_config} ${operation_timeout} + [Documentation] 1000 Flows deleted via Follower Node1 and verify it gets applied in all instances. + BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_del} ${operation_timeout} Verify No Flows In Cluster After Follower Restart - BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${get_small_config} ${operation_timeout} ${flow_count_after_del} + [Documentation] Verify flow count is 0 across cluster nodes. + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} diff --git a/csit/suites/openflowplugin/Clustering_Bulkomatic/030__Cluster_Reconcilliation_Multi_DPN.robot b/csit/suites/openflowplugin/Clustering_Bulkomatic/030__Cluster_Reconcilliation_Multi_DPN.robot new file mode 100644 index 0000000000..ed3134d2d1 --- /dev/null +++ b/csit/suites/openflowplugin/Clustering_Bulkomatic/030__Cluster_Reconcilliation_Multi_DPN.robot @@ -0,0 +1,115 @@ +*** Settings *** +Documentation Test suite for Cluster with Bulk Flows - Reconcilliation in a multi DPN environment +Suite Setup Create Controller Sessions +Suite Teardown Delete All Sessions +Resource ../../../libraries/BulkomaticKeywords.robot +Resource ../../../libraries/MininetKeywords.robot +Resource ../../../libraries/ClusterKeywords.robot +Resource ../../../libraries/ClusterOpenFlow.robot +Resource ../../../libraries/Utils.robot +Variables ../../../variables/Variables.py + +*** Variables *** +${operation_timeout} 100s +${flow_count_per_switch} 1000 +${switch_count} 3 +${flow_count_after_add} 3000 +${flow_count_after_del} 0 +${orig_json_config_add} sal_add_bulk_flow_config.json +${orig_json_config_get} sal_get_bulk_flow_config.json +${orig_json_config_del} sal_del_bulk_flow_config.json + +*** Test Cases *** +Create Original Cluster List + [Documentation] Create original cluster list. + ${original_cluster_list} ClusterKeywords.Create Controller Index List + Set Suite Variable ${original_cluster_list} + +Check Shards Status Before Leader Restart + [Documentation] Check Status for all shards in OpenFlow application. + ClusterOpenFlow.Check OpenFlow Shards Status ${original_cluster_list} + +Get Inventory Follower Before Cluster Restart + [Documentation] Find a follower in the inventory config shard + ${inventory_leader} ${inventory_followers} ClusterOpenFlow.Get InventoryConfig Shard Status ${original_cluster_list} + ${Follower_Node_1}= Get From List ${Inventory_Followers} 0 + ${Follower_Node_2}= Get From List ${Inventory_Followers} 1 + Set Suite Variable ${Follower_Node_1} + Set Suite Variable ${Follower_Node_2} + Set Suite Variable ${Inventory_Leader} + +Start Mininet Connect To Follower Node1 + [Documentation] Start mininet with connection to Follower Node1. + ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP} --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13 + Set Suite Variable ${mininet_conn_id} + +Add Bulk Flow From Follower + [Documentation] 3000 Flows (1K per DPN) in 3 DPN added via Follower Node1 and verify it gets applied in all instances. + ${temp_json_config_add} BulkomaticKeywords.Set DPN And Flow Count In Json Add ${orig_json_config_add} ${switch_count} ${flow_count_per_switch} + BulkomaticKeywords.Add Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_add} ${operation_timeout} + +Get Bulk Flows and Verify In Cluster + [Documentation] Initiate get operation and check flow count across cluster nodes + ${temp_json_config_get} BulkomaticKeywords.Set DPN And Flow Count In Json Get ${orig_json_config_get} ${switch_count} ${flow_count_after_add} + Set Suite Variable ${temp_json_config_get} + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_add} + +Verify Flows In Switch Before Cluster Restart + [Documentation] Verify flows are installed in switch before cluster restart. + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} + +Stop Mininet Connected To Follower Node1 and Exit + [Documentation] Stop mininet and exit connection. + MininetKeywords.Stop Mininet And Exit ${mininet_conn_id} + Utils.Clean Mininet System + +Start Mininet Reconnect To Follower Node1 + [Documentation] Start mininet with reconnection to follower node1. + ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_1}_IP} --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13 + Set Suite Variable ${mininet_conn_id} + +Verify Flows In Switch Reconnected To Follower Node1 + [Documentation] Verify 1K flows per DPN installed in switch after it is reconnected to follower node1. + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} + +Stop Mininet Connected To Follower Node1 + [Documentation] Stop mininet and exit connection. + MininetKeywords.Stop Mininet And Exit ${mininet_conn_id} + Utils.Clean Mininet System + +Start Mininet Connect To Follower Node2 + [Documentation] Start mininet with connection to follower node2. + ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Follower_Node_2}_IP} --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13 + Set Suite Variable ${mininet_conn_id} + +Verify Flows In Switch Connected To Follower Node2 + [Documentation] Verify 1K flows per DPN installed in switch after it is connected to follower node2. + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} + +Stop Mininet Connected To Follower Node2 + [Documentation] Stop mininet and exit connection. + MininetKeywords.Stop Mininet And Exit ${mininet_conn_id} + Utils.Clean Mininet System + +Start Mininet Connect To Inventory Leader + [Documentation] Start mininet with connection to inventroy leader. + ${mininet_conn_id}= MininetKeywords.Start Mininet Single Controller ${TOOLS_SYSTEM_IP} ${ODL_SYSTEM_${Inventory_Leader}_IP} --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13 + Set Suite Variable ${mininet_conn_id} + +Verify Flows In Switch Connected To Leader + [Documentation] Verify 1K flows per DPN installed in switch after it is connected to inventory leader. + MininetKeywords.Verify Aggregate Flow From Mininet Session ${mininet_conn_id} ${switch_count} ${flow_count_per_switch} ${operation_timeout} + +Stop Mininet Connected To Inventory Leader + [Documentation] Stop mininet and exit connection. + MininetKeywords.Stop Mininet And Exit ${mininet_conn_id} + Utils.Clean Mininet System + +Delete All Flows From Follower Node1 + [Documentation] 3000 Flows deleted via Follower Node1 and verify it gets applied in all instances. + ${temp_json_config_del} BulkomaticKeywords.Set DPN And Flow Count In Json Del ${orig_json_config_del} ${switch_count} ${flow_count_per_switch} + BulkomaticKeywords.Delete Bulk Flow In Node ${Follower_Node_1} ${temp_json_config_del} ${operation_timeout} + +Verify No Flows In Cluster + [Documentation] Verify flow count is 0 across cluster nodes. + BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster ${original_cluster_list} ${temp_json_config_get} ${operation_timeout} ${flow_count_after_del} diff --git a/csit/variables/openflowplugin/sal_add_bulk_flow_small_config.json b/csit/variables/openflowplugin/sal_add_bulk_flow_config.json similarity index 90% rename from csit/variables/openflowplugin/sal_add_bulk_flow_small_config.json rename to csit/variables/openflowplugin/sal_add_bulk_flow_config.json index 3a5a6cbedd..f745108fc2 100644 --- a/csit/variables/openflowplugin/sal_add_bulk_flow_small_config.json +++ b/csit/variables/openflowplugin/sal_add_bulk_flow_config.json @@ -6,7 +6,7 @@ "sal-bulk-flow:flows-per-dpn" : "1000", "sal-bulk-flow:batch-size" : "1", "sal-bulk-flow:seq" : "true", - "sal-bulk-flow:tx-chain" : "false", + "sal-bulk-flow:tx-chain" : "true", "sal-bulk-flow:sleep-for" : "0", "sal-bulk-flow:sleep-after" : "1", "sal-bulk-flow:start-table-id" : "0", diff --git a/csit/variables/openflowplugin/sal_del_bulk_flow_small_config.json b/csit/variables/openflowplugin/sal_del_bulk_flow_config.json similarity index 90% rename from csit/variables/openflowplugin/sal_del_bulk_flow_small_config.json rename to csit/variables/openflowplugin/sal_del_bulk_flow_config.json index 887df10df8..7f4dbcff3c 100644 --- a/csit/variables/openflowplugin/sal_del_bulk_flow_small_config.json +++ b/csit/variables/openflowplugin/sal_del_bulk_flow_config.json @@ -6,7 +6,7 @@ "sal-bulk-flow:flows-per-dpn" : "1000", "sal-bulk-flow:batch-size" : "1", "sal-bulk-flow:seq" : "true", - "sal-bulk-flow:tx-chain" : "false", + "sal-bulk-flow:tx-chain" : "true", "sal-bulk-flow:sleep-for" : "0", "sal-bulk-flow:sleep-after" : "1", "sal-bulk-flow:start-table-id" : "0", diff --git a/csit/variables/openflowplugin/sal_get_bulk_flow_small_config.json b/csit/variables/openflowplugin/sal_get_bulk_flow_config.json similarity index 100% rename from csit/variables/openflowplugin/sal_get_bulk_flow_small_config.json rename to csit/variables/openflowplugin/sal_get_bulk_flow_config.json