Stabilize OF cluster suite

Wait until switch shows up before adding flow.
Harden entity owner cluster suites
Replace Kill by Shutdown in most suites.
Remove check for master not changed after instance rejoin.
Place Shutdown Master instance test at the top.

Change-Id: I757670e88624178f7e76f82e95604d901aebf45b
Signed-off-by: Luis Gomez <ecelgp@gmail.com>

diff --git a/csit/suites/openflowplugin/Clustering/030__Cluster_HA_Data_Recovery_Leader_Follower_Failover.robot b/csit/suites/openflowplugin/Clustering/030__Cluster_HA_Data_Recovery_Leader_Follower_Failover.robot
index cce79971b8922b52a92c688829958404137b87bc..b220cfba87dc5d063304594ddcb2f95925642489 100644 (file)
--- a/csit/suites/openflowplugin/Clustering/030__Cluster_HA_Data_Recovery_Leader_Follower_Failover.robot
+++ b/csit/suites/openflowplugin/Clustering/030__Cluster_HA_Data_Recovery_Leader_Follower_Failover.robot
@@ -31,6 +31,8 @@ Get inventory Leader Before Leader Restart
 Start Mininet Connect To Follower Node1
     [Documentation]    Start mininet with connection to Follower Node1.
     ${mininet_conn_id}=    MininetKeywords.Start Mininet Single Controller    ${TOOLS_SYSTEM_IP}    ${ODL_SYSTEM_${follower_node_1}_IP}
+    Comment    Wait until switch is available in controller
+    Wait Until Keyword Succeeds    5s    1s    ClusterOpenFlow.Verify Switch Connections Running On Member    1    ${follower_node_1}
     Set Suite Variable    ${mininet_conn_id}
 
 Add Flows In Follower Node2 and Verify Before Leader Restart
@@ -43,8 +45,8 @@ Stop Mininet Connected To Follower Node1 and Exit
     Utils.Clean Mininet System
 
 Restart Leader From Cluster Node
-    [Documentation]    Kill Leader Node and Start it Up, Verify it is sync with other controller node.
-    ClusterManagement.Kill Single Member    ${inventory_leader}
+    [Documentation]    Stop Leader Node and Start it Up, Verify it is sync with other controller node.
+    ClusterManagement.Stop Single Member    ${inventory_leader}
     ClusterManagement.Start Single Member    ${inventory_leader}
 
 Get inventory Follower After Leader Restart
@@ -71,8 +73,8 @@ Stop Mininet Connected To Old Leader and Exit
     Utils.Clean Mininet System
 
 Restart Follower Node2
-    [Documentation]    Kill Follower Node2 and Start it Up, Verify it is sync with other controller node.
-    ClusterManagement.Kill Single Member    ${follower_node_2}
+    [Documentation]    Stop Follower Node2 and Start it Up, Verify it is sync with other controller node.
+    ClusterManagement.Stop Single Member    ${follower_node_2}
     ClusterManagement.Start Single Member    ${follower_node_2}
 
 Get inventory Follower After Follower Restart
@@ -99,8 +101,8 @@ Stop Mininet Connected To Leader and Exit
     Utils.Clean Mininet System
 
 Restart Full Cluster
-    [Documentation]    Kill all Cluster Nodes and Start it Up All.
-    ClusterManagement.Kill_Members_From_List_Or_All
+    [Documentation]    Stop all Cluster Nodes and Start it Up All.
+    ClusterManagement.Stop_Members_From_List_Or_All
     ClusterManagement.Start_Members_From_List_Or_All
 
 Get inventory Status After Cluster Restart

diff --git a/csit/suites/openflowplugin/EntityOwnership/010_Switch_Disconnect.robot b/csit/suites/openflowplugin/EntityOwnership/010_Switch_Disconnect.robot
index 60c52b9c3559d0341a266118333736c14e1463a3..9c24afe6e3d23bf3bb792770902044fd6d481153 100644 (file)
--- a/csit/suites/openflowplugin/EntityOwnership/010_Switch_Disconnect.robot
+++ b/csit/suites/openflowplugin/EntityOwnership/010_Switch_Disconnect.robot
@@ -69,6 +69,7 @@ Check All Switches Connected To All Cluster Nodes
 Reconnecting Switch Scenario
     [Arguments]    ${switch_name}
     [Documentation]    Disconnect and connect master and slave and check switch data to be consistent
+    BuiltIn.Set Test Variable    ${disc_cntl}    ${Empty}
     ${idx}=    BuiltIn.Evaluate    str("${switch_name}"[1:])
     BuiltIn.Set Test Variable    ${idx}
     Disconnect Switchs Old Master    ${switch_name}
@@ -79,13 +80,12 @@ Reconnecting Switch Scenario
 
 Disconnect Switchs Old Master
     [Arguments]    ${switch_name}
-    BuiltIn.Set Test Variable    ${disc_cntl}    ${Empty}
     ${old_owner}    ${old_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}
     ${old_master}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_owner}_IP}
     OvsManager.Disconnect Switch From Controller And Verify Disconnected    ${switch_name}    ${old_master}
     BuiltIn.Set Test Variable    ${disc_cntl}    ${old_master}
-    ${new_master}=    BuiltIn.Wait Until Keyword Succeeds    5x    3s    Verify New Master Controller Node    ${switch_name}    ${old_master}
     ${owner}    ${successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${old_successors}    after_stop=True
+    ${new_master}=    BuiltIn.Wait Until Keyword Succeeds    5x    3s    Verify New Master Controller Node    ${switch_name}    ${old_master}
     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
     BuiltIn.Set Test Variable    ${old_owner}
     BuiltIn.Set Test Variable    ${old_successors}

diff --git a/csit/suites/openflowplugin/EntityOwnership/020_Cluster_Node_Failure.robot b/csit/suites/openflowplugin/EntityOwnership/020_Cluster_Node_Failure.robot
index 9b11d8cd7270da53370cbfbc3d93f63801406073..e5c460211ed9caf8201dc441d8664a6c20a786f7 100644 (file)
--- a/csit/suites/openflowplugin/EntityOwnership/020_Cluster_Node_Failure.robot
+++ b/csit/suites/openflowplugin/EntityOwnership/020_Cluster_Node_Failure.robot
@@ -88,7 +88,7 @@ Kill Switchs Old Owner
     BuiltIn.Set Test Variable    ${stopped_karaf}    ${old_owner}
     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${old_successors}
     ${new_master}=    BuiltIn.Wait Until Keyword Succeeds    5x    3s    Verify New Master Controller Node    ${switch_name}    ${old_master}
-    ${owner}    ${successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${old_successors}
+    ${owner}    ${successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${old_successors}    after_stop=True
     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
     BuiltIn.Set Suite Variable    ${active_member}    ${owner}
     BuiltIn.Set Test Variable    ${old_owner}
@@ -115,7 +115,7 @@ Kill Switchs Successor
     ${tmp_candidates}=    BuiltIn.Create List    @{ClusterManagement__member_index_list}
     Collections.Remove Values From List    ${tmp_candidates}    ${old_successor}
     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${tmp_candidates}
-    ${owner}    ${successor}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${tmp_candidates}
+    ${owner}    ${successor}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${tmp_candidates}    after_stop=True
     BuiltIn.Should Be Equal    ${owner}    ${old_owner}
     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
     BuiltIn.Set Test Variable    ${old_owner}
@@ -144,7 +144,7 @@ Verify New Master Controller Node
 Stop Controller Node And Verify
     [Arguments]    ${node}
     [Documentation]    Stops the given node
-    ClusterManagement.Kill Single Member    ${node}
+    ClusterManagement.Stop Single Member    ${node}
     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}
 
 Start Controller Node And Verify

diff --git a/csit/suites/openflowplugin/EntityOwnership/030_Cluster_Sync_Problems.robot b/csit/suites/openflowplugin/EntityOwnership/030_Cluster_Sync_Problems.robot
index 4f3791a7fd941a8702f7fb08421d2890cb3bc097..495e3f9fd802b8dafd74ef3ef134868a002d9012 100644 (file)
--- a/csit/suites/openflowplugin/EntityOwnership/030_Cluster_Sync_Problems.robot
+++ b/csit/suites/openflowplugin/EntityOwnership/030_Cluster_Sync_Problems.robot
@@ -81,6 +81,7 @@ Check All Switches Connected To All Cluster Nodes
 Isolating Node Scenario
     [Arguments]    ${switch_name}
     [Documentation]    Disconnect and connect owner and successor and check switch data to be consistent
+    BuiltIn.Set Test Variable    ${isol_node}    ${Empty}
     ${idx}=    BuiltIn.Evaluate    str("${switch_name}"[1:])
     BuiltIn.Set Test Variable    ${idx}
     Isolate Switchs Old Owner    ${switch_name}
@@ -91,7 +92,6 @@ Isolating Node Scenario
 
 Isolate Switchs Old Owner
     [Arguments]    ${switch_name}
-    BuiltIn.Set Test Variable    ${isol_node}    ${Empty}
     ${old_owner}    ${old_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}
     ${old_master}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_owner}_IP}
     ${active_member}=    Collections.Get From List    ${old_successors}    0
@@ -123,13 +123,12 @@ Isolate Switchs Successor
     ${old_successor}=    Collections.Get From List    ${old_successors}    0
     ${old_slave}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_successor}_IP}
     Isolate Controller From The Cluster    ${old_successor}
-    BuiltIn.Set Test Variable    ${isol_cntl}    ${old_slave}
+    BuiltIn.Set Test Variable    ${isol_node}    ${old_successor}
     ${tmp_candidates}=    BuiltIn.Create List    @{ClusterManagement__member_index_list}
     Collections.Remove Values From List    ${tmp_candidates}    ${old_successor}
     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${tmp_candidates}
     ${owner}    ${successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${tmp_candidates}
     BuiltIn.Should Be Equal    ${owner}    ${old_owner}
-    BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
     BuiltIn.Set Test Variable    ${old_owner}
     BuiltIn.Set Test Variable    ${old_successors}
     BuiltIn.Set Test Variable    ${old_successor}

diff --git a/csit/testplans/openflowplugin-clustering.txt b/csit/testplans/openflowplugin-clustering.txt
index 102b6620ed73dcab3c153a31cbbb4c8f8b0ceb06..bbe5b3d794e30a0fba61485d9a6a49b9c3bb51e5 100644 (file)
--- a/csit/testplans/openflowplugin-clustering.txt
+++ b/csit/testplans/openflowplugin-clustering.txt
@@ -1,6 +1,6 @@
 # Place the suites in run order:
-integration/test/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
 integration/test/csit/suites/openflowplugin/Clustering/020__Cluster_HA_Owner_Restart.robot
+integration/test/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
 integration/test/csit/suites/openflowplugin/Clustering/030__Cluster_HA_Data_Recovery_Leader_Follower_Failover.robot
 integration/test/csit/suites/openflowplugin/Clustered_Reconciliation/010_Group_Flows.robot
 integration/test/csit/suites/openflowplugin/EntityOwnership/010_Switch_Disconnect.robot