Enhance netvirt l2 HA tests

This was reverted since sandbox run was showing some failures.
Fixing the issues seen in the sandbox run, and putting this back.

Change-Id: Idea1fb5573a79ec3a17c3fad22108241b25f8689
Signed-off-by: Faseela K <faseela.k@ericsson.com>

diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot
index 89d2f3371115e28b37f7f0a95058f4882226805e..cce32302ae99f29db1d6b7f0d17dc96ee47a472d 100644 (file)
--- a/csit/libraries/ClusterManagement.robot
+++ b/csit/libraries/ClusterManagement.robot
@@ -439,13 +439,14 @@ Stop_Members_From_List_Or_All
     [Return]    ${updated_index_list}
 
 Start_Single_Member
-    [Arguments]    ${member}    ${wait_for_sync}=True    ${timeout}=300s
+    [Arguments]    ${member}    ${wait_for_sync}=True    ${timeout}=300s    ${check_system_status}=False    ${service_list}=@{EMPTY}
     [Documentation]    Convenience keyword that starts the specified member of the cluster.
     ${index_list} =    ClusterManagement__Build_List    ${member}
-    Start_Members_From_List_Or_All    ${index_list}    ${wait_for_sync}    ${timeout}
+    Start_Members_From_List_Or_All    ${index_list}    ${wait_for_sync}    ${timeout}    check_system_status=${check_system_status}    service_list=@{service_list}
 
 Start_Members_From_List_Or_All
     [Arguments]    ${member_index_list}=${EMPTY}    ${wait_for_sync}=True    ${timeout}=300s    ${karaf_home}=${EMPTY}    ${export_java_home}=${EMPTY}    ${gc_log_dir}=${EMPTY}
+    ...    ${check_system_status}=False    ${service_list}=@{EMPTY}
     [Documentation]    If the list is empty, start all cluster members. Otherwise, start members based on present indices.
     ...    If ${wait_for_sync}, wait for cluster sync on listed members.
     ...    Optionally karaf_home can be overriden. Optionally specific JAVA_HOME is used for starting.
@@ -458,7 +459,8 @@ Start_Members_From_List_Or_All
     Run_Bash_Command_On_List_Or_All    command=${command} ${gc_options}    member_index_list=${member_index_list}
     BuiltIn.Return_From_Keyword_If    not ${wait_for_sync}
     BuiltIn.Wait_Until_Keyword_Succeeds    ${timeout}    10s    Check_Cluster_Is_In_Sync    member_index_list=${member_index_list}
-    # TODO: Do we also want to check Shard Leaders here?
+    BuiltIn.Return_From_Keyword_If    not ${check_system_status}
+    CompareStream.Run_Keyword_If_At_Least_Oxygen    Wait Until Keyword Succeeds    60    2    ClusterManagement.Check Status of Services    @{service_list}
     [Teardown]    Run_Bash_Command_On_List_Or_All    command=netstat -pnatu | grep 2550
 
 Freeze_Single_Member

diff --git a/csit/suites/openstack/clustering/ha_l2.robot b/csit/suites/openstack/clustering/ha_l2.robot
index c40b593f06fc153b8f15bbac88f336ac6ad18547..50f8de51bc6663ff483f9bbf9c872fe98ca29ce9 100644 (file)
--- a/csit/suites/openstack/clustering/ha_l2.robot
+++ b/csit/suites/openstack/clustering/ha_l2.robot
@@ -16,6 +16,7 @@ Resource          ../../../libraries/ClusterOvsdb.robot
 Resource          ../../../libraries/ClusterManagement.robot
 Resource          ../../../libraries/SetupUtils.robot
 Resource          ../../../variables/Variables.robot
+Resource          ../../../variables/netvirt/Variables.robot
 
 *** Variables ***
 ${SECURITY_GROUP}    cl2_sg
@@ -82,7 +83,7 @@ Delete the Bridge Manually and Verify After Fail
 
 Bring Up ODL1
     [Documentation]    Bring up ODL1 again
-    ClusterManagement.Start Single Member    1
+    ClusterManagement.Start Single Member    member=1    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Create Bridge Manually and Verify After Recover
     [Documentation]    Create bridge with OVS command and verify it gets applied from all instances.
@@ -126,7 +127,7 @@ Check Vm Instances Have Ip Address
 
 Bring Up ODL2
     [Documentation]    Bring up ODL2 again
-    ClusterManagement.Start Single Member    2
+    ClusterManagement.Start Single Member    member=2    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Ping Vm Instance1 In net_1
     [Documentation]    Check reachability of vm instances by pinging to them.
@@ -170,7 +171,7 @@ Connectivity Tests From Vm Instance3 In net_1
 
 Bring Up ODL3
     [Documentation]    Bring up ODL3 again
-    ClusterManagement.Start Single Member    3
+    ClusterManagement.Start Single Member    member=3    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Take Down ODL1 and ODL2
     [Documentation]    Kill the karaf in First and Second Controller
@@ -190,7 +191,7 @@ Connectivity Tests From Vm Instance3 In net_2
 
 Bring Up ODL1 and ODL2
     [Documentation]    Bring up ODL1 and ODL2 again.
-    ClusterManagement.Start Members From List Or All    ${CLUSTER_DOWN_LIST}
+    ClusterManagement.Start Members From List Or All    member_index_list=${CLUSTER_DOWN_LIST}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Delete Vm Instance
     [Documentation]    Delete Vm instances using instance names. Also remove the VM from the

diff --git a/csit/suites/openstack/clustering/ha_l3.robot b/csit/suites/openstack/clustering/ha_l3.robot
index b97b908d3d7d5ed4f1b6059ce291f76da7a076f1..412eb510f50c6c978572ff04906814adea358281 100644 (file)
--- a/csit/suites/openstack/clustering/ha_l3.robot
+++ b/csit/suites/openstack/clustering/ha_l3.robot
@@ -16,6 +16,7 @@ Resource          ../../../libraries/ClusterOvsdb.robot
 Resource          ../../../libraries/ClusterManagement.robot
 Resource          ../../../libraries/SetupUtils.robot
 Resource          ../../../variables/Variables.robot
+Resource          ../../../variables/netvirt/Variables.robot
 
 *** Variables ***
 ${SECURITY_GROUP}    cl3_sg
@@ -55,7 +56,7 @@ Create Subnets For net_2
 
 Bring Up Leader Of Default Shard
     [Documentation]    Bring up on cluster leader
-    ClusterManagement.Start Single Member    ${cluster_leader}
+    ClusterManagement.Start Single Member    member=${cluster_leader}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Add Ssh Allow All Rule
     [Documentation]    Allow all TCP/UDP/ICMP packets for this suite
@@ -73,7 +74,7 @@ Create Vm Instances For net_1
 
 Bring Up ODL1
     [Documentation]    Bring up ODL1 again
-    ClusterManagement.Start Single Member    1
+    ClusterManagement.Start Single Member    member=1    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Take Down ODL2
     [Documentation]    Kill the karaf in Second Controller
@@ -99,7 +100,7 @@ Check Vm Instances Have Ip Address
 
 Bring Up ODL2
     [Documentation]    Bring up ODL2 again
-    ClusterManagement.Start Single Member    2
+    ClusterManagement.Start Single Member    member=2    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Take Down ODL3
     [Documentation]    Kill the karaf in Third Controller
@@ -127,7 +128,7 @@ Verify Created Routers
 
 Bring Up ODL3
     [Documentation]    Bring up ODL3 again
-    ClusterManagement.Start Single Member    3
+    ClusterManagement.Start Single Member    member=3    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Ping Vm Instance1 In net_2 From net_1
     [Documentation]    Check reachability of vm instances by pinging to them after creating routers.
@@ -178,7 +179,7 @@ Connectivity Tests From Vm Instance3 In net_1
 
 Bring Up ODL1 and ODL2
     [Documentation]    Bring up ODL1 and ODL2 again
-    ClusterManagement.Start Members From List Or All    ${ODL_1_AND_2_DOWN}
+    ClusterManagement.Start Members From List Or All    member_index_list=${ODL_1_AND_2_DOWN}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Take Down ODL2 and ODL3
     [Documentation]    Kill the karaf in First and Second Controller
@@ -205,7 +206,7 @@ Connectivity Tests From Vm Instance3 In net_2
 
 Bring Up ODL2 and ODL3
     [Documentation]    Bring up ODL2 and ODL3 again.
-    ClusterManagement.Start Members From List Or All    ${ODL_2_AND_3_DOWN}
+    ClusterManagement.Start Members From List Or All    member_index_list=${ODL_2_AND_3_DOWN}    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Take Down All Instances
     [Documentation]    Stop karaf on all controllers
@@ -213,7 +214,7 @@ Take Down All Instances
 
 Bring Up All Instances
     [Documentation]    Bring up all controllers
-    ClusterManagement.Start Members From List Or All
+    ClusterManagement.Start Members From List Or All    check_system_status=True    service_list=@{NETVIRT_DIAG_SERVICES}
 
 Connectivity Tests From Vm Instance2 In net_2 after recovering all nodes
     [Documentation]    ssh to the VM instance and test operations.

diff --git a/csit/variables/netvirt/Variables.robot b/csit/variables/netvirt/Variables.robot
index 9c30400d12d7c1853833f263edd446f67dcb6456..2e054754a88fb092a8ec67379e5286a4dfe93afd 100644 (file)
--- a/csit/variables/netvirt/Variables.robot
+++ b/csit/variables/netvirt/Variables.robot
@@ -4,6 +4,7 @@ Documentation     Variables for Netvirt Test Suites
 *** Variables ***
 # Exceptions for which we will not create a failure
 @{legacy_feature_list}    odl-vtn-manager-neutron    odl-ovsdb-openstack
+@{NETVIRT_DIAG_SERVICES}    OPENFLOW    IFM    ITM    DATASTORE    ELAN    OVSDB
 ${CIRROS_stable/ocata}    cirros-0.3.4-x86_64-uec
 ${CIRROS_stable/pike}    cirros-0.3.5-x86_64-disk
 ${CIRROS_stable/queens}    cirros-0.3.5-x86_64-disk