Verify Bulkomatic Flows with 2Node Cluster

Change-Id: Ie6be63d1d713311c8f0071a206e539c38fb77844
Signed-off-by: naveen <naveenk.t@tcs.com>

diff --git a/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot b/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot
index ecb6ba7ee3327e537232200bd8e8bfb48bbf9f89..7aedcc0f38fda6b9ce91ea85c949fabd6db9f5da 100644 (file)
--- a/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot
+++ b/csit/suites/openflowplugin/Clustering_Bulkomatic/020__Cluster_HA_Data_Recovery_BulkFlow_2Node_Cluster.robot
@@ -12,9 +12,9 @@ Variables         ../../../variables/Variables.py
 *** Variables ***
 ${operation_timeout}    100s
 ${restart_timeout}    350s
-${flow_count_per_switch}    1000
+${flow_count_per_switch}    10000
 ${switch_count}    1
-${flow_count_after_add}    1000
+${flow_count_after_add}    10000
 ${flow_count_after_del}    0
 ${orig_json_config_add}    sal_add_bulk_flow_config.json
 ${orig_json_config_get}    sal_get_bulk_flow_config.json
@@ -31,48 +31,61 @@ Check Shards Status and Initialize Variables
     Set Suite Variable    ${temp_json_config_get}
     Set Suite Variable    ${temp_json_config_del}
 
-Get Inventory Follower Before Cluster Restart
+Get Inventory Follower Before Leader Restart
     [Documentation]    Find a follower in the inventory config shard
     ${inventory_leader}    ${inventory_followers}    ClusterOpenFlow.Get InventoryConfig Shard Status
-    ${Inventory_Leader_List}=    Create List    ${Inventory_Leader}
+    ${Inventory_Pre_Leader_List}=    Create List    ${Inventory_Leader}
     ${Follower_Node_1}=    Get From List    ${Inventory_Followers}    0
     Set Suite Variable    ${Inventory_Followers}
     Set Suite Variable    ${Follower_Node_1}
     Set Suite Variable    ${Inventory_Leader}
-    Set Suite Variable    ${Inventory_Leader_List}
+    Set Suite Variable    ${Inventory_Pre_Leader_List}
+    ${Inventory_Pre_Leader}    Set Variable    ${Inventory_Leader}
+    Set Suite Variable    ${Inventory_Pre_Leader}
 
 Shutdown Leader From Cluster Node
-    [Documentation]    Shutdown Leader Node and Start it Up.
+    [Documentation]    Shutdown Leader Node and Start it Up
     ClusterManagement.Kill Single Member    ${Inventory_Leader}
 
 Check Shards Status After Leader Shutdown
     [Documentation]    Wait for node convergence and check status for all shards in OpenFlow application.
     Wait Until Keyword Succeeds    ${operation_timeout}    2s    ClusterOpenFlow.Check OpenFlow Shards Status    ${Inventory_Followers}
 
+Check Shard Status For Leader After PreLeader Shutdown
+    [Documentation]    Find a Leader in the inventory config shard
+    ${Inventory_Leader_Post}    ${inventory_followers}    ClusterOpenFlow.Get InventoryConfig Shard Status    ${Inventory_Followers}
+    ${Inventory_Leader_List_Post}=    Create List    ${Inventory_Leader_Post}
+    ${Follower_Node_1}=    Get From List    ${Inventory_Followers}    0
+    Set Suite Variable    ${Inventory_Followers}
+    Set Suite Variable    ${Follower_Node_1}
+    Set Suite Variable    ${Inventory_Leader_Post}
+    Set Suite Variable    ${Inventory_Leader_List_Post}
+
 Start Mininet Connect To Follower Node1
-    [Documentation]    Start mininet with connection to Follower Node1.
-    ${mininet_conn_id}=    MininetKeywords.Start Mininet Single Controller    ${TOOLS_SYSTEM_IP}    ${ODL_SYSTEM_${Follower_Node_1}_IP}
+    [Documentation]    Start mininet with connection to Follower Node1
+    ${mininet_conn_id}=    MininetKeywords.Start Mininet Single Controller    ${TOOLS_SYSTEM_IP}    ${ODL_SYSTEM_${Follower_Node_1}_IP}    --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13
     Set Suite Variable    ${mininet_conn_id}
 
 Add Bulk Flow From Follower
-    [Documentation]    1000 Flows added via Follower Node1 and verify it gets applied in all instances.
+    [Documentation]    ${flow_count_per_switch} Flows added via Follower Node1 and verify it gets applied in all instances.
     BulkomaticKeywords.Add Bulk Flow In Node    ${temp_json_config_add}    ${Follower_Node_1}    ${operation_timeout}
 
-Get Bulk Flows and Verify In Cluster
+Get Bulk Flows And Verify In Leader
     [Documentation]    Initiate get operation and check flow count across cluster nodes.
-    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}    ${Inventory_Followers}
+    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}    ${Inventory_Leader_List_Post}
 
 Verify Flows In Switch Before Cluster Restart
     [Documentation]    Verify flows are installed in switch before cluster restart.
     MininetKeywords.Verify Aggregate Flow From Mininet Session    ${mininet_conn_id}    ${flow_count_after_add}    ${operation_timeout}
 
-Restart Leader From Cluster Node
+Restart Pre Leader From Cluster Node
     [Documentation]    Restart Leader Node.
     ClusterManagement.Start Single Member    ${Inventory_Leader}
 
 Verify Data Recovery After Leader Restart
-    [Documentation]    1000 Flows preserved in all controller instances.
+    [Documentation]    ${flow_count_after_add} Flows preserved in all controller instances.
     Wait Until Keyword Succeeds    ${restart_timeout}    2s    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}
+    ...    ${Inventory_Leader_List_Post}
 
 Verify Flows In Switch After Leader Restart
     [Documentation]    Verify flows are installed in switch after cluster restart.
@@ -84,23 +97,28 @@ Stop Mininet Connected To Follower Node1
     Utils.Clean Mininet System
 
 Delete All Flows From Follower Node1
-    [Documentation]    1000 Flows deleted via Follower Node1 and verify it gets applied in all instances.
+    [Documentation]    ${flow_count_after_add} Flows deleted via Follower Node1 and verify it gets applied in all instances.
     BulkomaticKeywords.Delete Bulk Flow In Node    ${temp_json_config_del}    ${Follower_Node_1}    ${operation_timeout}
 
-Verify No Flows In Cluster
-    [Documentation]    Verify flow count is 0 across cluster nodes.
-    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_del}
+Verify No Flows In Leader Node
+    [Documentation]    Verify flow count is ${flow_count_after_del} across cluster nodes.
+    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_del}    ${Inventory_Leader_List_Post}
 
-Get Inventory Follower Before Follower Restart
-    [Documentation]    Find Leader and followers in the inventory config shard
+Get Inventory Follower And Leader Before Cluster Restart
+    [Documentation]    Find a follower in the inventory config shard
     ${inventory_leader}    ${inventory_followers}    ClusterOpenFlow.Get InventoryConfig Shard Status
     ${Active_Nodes}=    Create List
     ${Follower_Node_1}=    Get From List    ${Inventory_Followers}    0
     ${Follower_Node_2}=    Get From List    ${Inventory_Followers}    1
     Append To List    ${Active_Nodes}    ${inventory_leader}    ${Follower_Node_1}
     Set Suite Variable    ${Active_Nodes}
+    ${Inventory_Leader_List}=    Create List    ${inventory_leader}
+    ${Inventory_Follower_Node1_List}=    Create List    ${Follower_Node_1}
     Set Suite Variable    ${Follower_Node_1}
     Set Suite Variable    ${Follower_Node_2}
+    Set Suite Variable    ${Inventory_Leader}
+    Set Suite Variable    ${Inventory_Leader_List}
+    Set Suite Variable    ${Inventory_Follower_Node1_List}
 
 Shutdown Follower From Cluster Node
     [Documentation]    Shutdown Follower Node2 and Start it Up.
@@ -112,16 +130,16 @@ Check Shards Status After Follower Shutdown
 
 Start Mininet Connect To Follower Node
     [Documentation]    Start mininet with connection to Follower Node1.
-    ${mininet_conn_id}=    MininetKeywords.Start Mininet Single Controller    ${TOOLS_SYSTEM_IP}    ${ODL_SYSTEM_${Follower_Node_1}_IP}
+    ${mininet_conn_id}=    MininetKeywords.Start Mininet Single Controller    ${TOOLS_SYSTEM_IP}    ${ODL_SYSTEM_${Follower_Node_1}_IP}    --topo linear,${switch_count} --switch ovsk,protocols=OpenFlow13
     Set Suite Variable    ${mininet_conn_id}
 
 Add Bulk Flow From Follower Node1
-    [Documentation]    1000 Flows added via Follower Node1 and verify it gets applied in all instances.
+    [Documentation]    ${flow_count_after_add} Flows added via Follower Node1 and verify it gets applied in all instances.
     BulkomaticKeywords.Add Bulk Flow In Node    ${temp_json_config_add}    ${Follower_Node_1}    ${operation_timeout}
 
-Get Bulk Flows and Verify In Cluster Before Follower Restart
-    [Documentation]    Initiate get operation and check flow count only across active cluster nodes
-    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}    ${Active_Nodes}
+Get Bulk Flows And Verify In Leader Before Follower Restart
+    [Documentation]    Initiate get operation and check flow count ${flow_count_after_add} only across active cluster nodes
+    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}    ${Inventory_Leader_List}
 
 Verify Flows In Switch Before Follower Restart
     [Documentation]    Verify flows are installed in switch before follower restart.
@@ -132,8 +150,9 @@ Restart Follower From Cluster Node
     ClusterManagement.Start Single Member    ${Follower_Node_2}
 
 Verify Data Recovery After Follower Restart
-    [Documentation]    1000 Flows preserved in all controller instances.
+    [Documentation]    ${flow_count_after_add} Flows preserved in all controller instances.
     Wait Until Keyword Succeeds    ${restart_timeout}    2s    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_add}
+    ...    ${Inventory_Leader_List}
 
 Verify Flows In Switch After Follower Restart
     [Documentation]    Verify flows are installed in switch after cluster restart.
@@ -145,9 +164,9 @@ Stop Mininet Connected To Follower Node
     Utils.Clean Mininet System
 
 Delete All Flows From Follower Node
-    [Documentation]    1000 Flows deleted via Follower Node1 and verify it gets applied in all instances.
+    [Documentation]    ${flow_count_after_add} Flows deleted via Follower Node1 and verify it gets applied in all instances.
     BulkomaticKeywords.Delete Bulk Flow In Node    ${temp_json_config_del}    ${Follower_Node_1}    ${operation_timeout}
 
-Verify No Flows In Cluster After Follower Restart
-    [Documentation]    Verify flow count is 0 across cluster nodes.
-    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_del}
+Verify No Flows In Leader Node After Follower Restart
+    [Documentation]    Verify flow count is ${flow_count_after_del} across cluster nodes.
+    BulkomaticKeywords.Get Bulk Flow And Verify Count In Cluster    ${temp_json_config_get}    ${operation_timeout}    ${flow_count_after_del}    ${Inventory_Leader_List}