NETVIRT-1572

Signed-off-by: Srinivas Rachakonda <srinivas.rachakonda@gmail.com>
Change-Id: Ifc33be3605d080ee3dea2eb733f3b1d5412b2834
Signed-off-by: Srinivas Rachakonda <srinivas.rachakonda@gmail.com>

diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot
index 92c861e1f42cb5e3d4d6bdcc5338c6ff2e0bd2f3..1c84caa9ad0c805485d45588d860aba59737d9f4 100644 (file)
--- a/csit/libraries/ClusterManagement.robot
+++ b/csit/libraries/ClusterManagement.robot
@@ -153,6 +153,15 @@ Get_Raft_State_Of_Shard_At_Member
     ...    http_timeout=${http_timeout}
     [Return]    ${raft_state}
 
+Get_Raft_State_Of_Shard_Of_All_Member_Nodes
+    [Arguments]    ${shard_name}=default    ${shard_type}=config    ${member_index_list}=${EMPTY}
+    [Documentation]    Get raft state of shard of all member nodes
+    ${index_list} =    List_Indices_Or_All    given_list=${member_index_list}
+    Collections.Sort_List    ${index_list}
+    FOR    ${index}    IN    @{index_list}
+        ClusterManagement.Get Raft State Of Shard At Member    shard_name=${shard_name}    shard_type=${shard_type}    member_index=${index}
+    END
+
 Get_Raft_Property_From_Shard_Member
     [Arguments]    ${property}    ${shard_name}    ${shard_type}    ${member_index}    ${verify_restconf}=False    ${http_timeout}=${EMPTY}
     [Documentation]    Send request to Jolokia on indexed member, return extracted Raft property.
@@ -438,7 +447,7 @@ Stop_Single_Member
     [Return]    ${updated_index_list}
 
 Stop_Members_From_List_Or_All
-    [Arguments]    ${member_index_list}=${EMPTY}    ${original_index_list}=${EMPTY}    ${confirm}=True    ${timeout}=240s
+    [Arguments]    ${member_index_list}=${EMPTY}    ${original_index_list}=${EMPTY}    ${confirm}=True    ${timeout}=360s
     [Documentation]    If the list is empty, stops all ODL instances. Otherwise stop members based on \${stop_index_list}
     ...    If \${confirm} is True, verify stopped instances are not there anymore.
     ...    The KW will return a list of available members: \${updated index_list}=\${original_index_list}-\${member_index_list}
@@ -465,7 +474,7 @@ Start_Single_Member
     Start_Members_From_List_Or_All    ${index_list}    ${wait_for_sync}    ${timeout}    check_system_status=${check_system_status}    verify_restconf=${verify_restconf}    service_list=${service_list}
 
 Start_Members_From_List_Or_All
-    [Arguments]    ${member_index_list}=${EMPTY}    ${wait_for_sync}=True    ${timeout}=300s    ${karaf_home}=${EMPTY}    ${export_java_home}=${EMPTY}    ${gc_log_dir}=${EMPTY}
+    [Arguments]    ${member_index_list}=${EMPTY}    ${wait_for_sync}=True    ${timeout}=360s    ${karaf_home}=${EMPTY}    ${export_java_home}=${EMPTY}    ${gc_log_dir}=${EMPTY}
     ...    ${check_system_status}=False    ${verify_restconf}=True    ${service_list}=${EMPTY_LIST}
     [Documentation]    If the list is empty, start all cluster members. Otherwise, start members based on present indices.
     ...    If ${wait_for_sync}, wait for cluster sync on listed members.

diff --git a/csit/suites/openstack/clustering/ha_l3.robot b/csit/suites/openstack/clustering/ha_l3.robot
index 9b810bb70551853481b7819033fc862a17f25211..2fa9dbcf4c575c533f9d590b0e64887c2267696e 100644 (file)
--- a/csit/suites/openstack/clustering/ha_l3.robot
+++ b/csit/suites/openstack/clustering/ha_l3.robot
@@ -29,18 +29,22 @@ ${SECURITY_GROUP}    cl3_sg
 @{GATEWAY_IPS}    36.0.0.1    37.0.0.1
 @{ODL_1_AND_2_DOWN}    ${1}    ${2}
 @{ODL_2_AND_3_DOWN}    ${2}    ${3}
+@{index_list}     1    2    3
 
 *** Test Cases ***
 Create All Controller Sessions
     [Documentation]    Create sessions for all three controllers.
     ClusterManagement.ClusterManagement Setup
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Take Down Leader Of Default Shard
     [Documentation]    Stop the karaf on ODL cluster leader
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ${cluster_leader}    ${followers} =    ClusterManagement.Get Leader And Followers For Shard    shard_type=config
     BuiltIn.Set Suite Variable    ${cluster_leader}
     ${new_cluster_list} =    ClusterManagement.Stop Single Member    ${cluster_leader}    msg=up: ODL1, ODL2, ODL3, down=none
     BuiltIn.Set Suite Variable    ${new_cluster_list}
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Create Networks
     [Documentation]    Create Network with neutron request.
@@ -59,6 +63,7 @@ Create Subnets For net_2
 Bring Up Leader Of Default Shard
     [Documentation]    Bring up on cluster leader
     ClusterManagement.Start Single Member    ${cluster_leader}    msg=up: ${new_cluster_list}, down: ${cluster_leader}
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Add Ssh Allow All Rule
     [Documentation]    Allow all TCP/UDP/ICMP packets for this suite
@@ -66,7 +71,9 @@ Add Ssh Allow All Rule
 
 Take Down ODL1
     [Documentation]    Stop the karaf in First Controller
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Create Vm Instances For net_1
     [Documentation]    Create Vm instances using flavor and image names for a network.
@@ -76,11 +83,15 @@ Create Vm Instances For net_1
 
 Bring Up ODL1
     [Documentation]    Bring up ODL1 again
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    1    msg=up: ODL2, ODL3, down: ODL1
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Take Down ODL2
     [Documentation]    Stop the karaf in Second Controller
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Stop Single Member    2    msg=up: ODL1, ODL2, ODL3, down=none
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Create Vm Instances For net_2
     [Documentation]    Create Vm instances using flavor and image names for a network.
@@ -102,11 +113,15 @@ Check Vm Instances Have Ip Address
 
 Bring Up ODL2
     [Documentation]    Bring up ODL2 again
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    2    msg=up: ODL1, ODL3, down: ODL2
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Take Down ODL3
     [Documentation]    Stop the karaf in Third Controller
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Stop Single Member    3    msg=up: ODL1, ODL2, ODL3, down=none
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Create Router router_2
     [Documentation]    Create Router and Add Interface to the subnets.
@@ -131,7 +146,9 @@ Verify Created Routers
 
 Bring Up ODL3
     [Documentation]    Bring up ODL3 again
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    3    msg=up: ODL1, ODL2, down: ODL3
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
 
 Ping Vm Instance1 In net_2 From net_1
     [Documentation]    Check reachability of vm instances by pinging to them after creating routers.
@@ -177,8 +194,10 @@ Connectivity Tests From Vm Instance3 In net_1 In Healthy Cluster
 
 Take Down ODL1 and ODL2
     [Documentation]    Stop the karaf in First and Second Controller
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
     ClusterManagement.Stop Single Member    2    msg=up: ODL2, ODL3, down=ODL1
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance1 In net_1 With Two ODLs Down
@@ -202,54 +221,63 @@ Connectivity Tests From Vm Instance3 In net_1 With Two ODLs Down
 Bring Up ODL1 and ODL2
     [Documentation]    Bring up ODL1 and ODL2 again. Do not check for cluster sync until all nodes are
     ...    up. akka will not let nodes join until they are all back up if two were down.
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    1    msg=up: ODL3, down: ODL1, ODL2    wait_for_sync=False
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    2    msg=up: ODL1, ODL3, down: ODL2
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Take Down ODL2 and ODL3
     [Documentation]    Stop the karaf in First and Second Controller
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Stop Single Member    2    msg=up: ODL1, ODL2, ODL3, down=none
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Stop Single Member    3    msg=up: ODL1, ODL3, down=ODL2
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance1 In net_2
     [Documentation]    ssh to the VM instance and test operations.
     ${dst_list} =    BuiltIn.Create List    @{NET_2_L3_VM_IPS}    @{NET_1_L3_VM_IPS}
-    OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_L3_VM_IPS}[0]    ${dst_list}
+    BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_L3_VM_IPS}[0]    ${dst_list}
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance2 In net_2
     [Documentation]    ssh to the VM instance and test operations.
     ${dst_list} =    BuiltIn.Create List    @{NET_2_L3_VM_IPS}    @{NET_1_L3_VM_IPS}
-    OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_L3_VM_IPS}[1]    ${dst_list}
+    BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_L3_VM_IPS}[1]    ${dst_list}
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance3 In net_2
     [Documentation]    ssh to the VM instance and test operations.
     ${dst_list} =    BuiltIn.Create List    @{NET_2_L3_VM_IPS}    @{NET_1_L3_VM_IPS}
-    OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_L3_VM_IPS}[2]    ${dst_list}
+    BuiltIn.Wait Until Keyword Succeeds    30s    10s    OpenStackOperations.Test Operations From Vm Instance    @{NETWORKS}[1]    @{NET_2_L3_VM_IPS}[2]    ${dst_list}
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Bring Up ODL2 and ODL3
     [Documentation]    Bring up ODL2 and ODL3 again. Do not check for cluster sync until all nodes are
     ...    up. akka will not let nodes join until they are all back up if two were down.
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    2    msg=up: ODL1, down: ODL2, ODL3    wait_for_sync=False
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     ClusterManagement.Start Single Member    3    msg=up: ODL1, ODL2, down: ODL3
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Take Down All Instances
     [Documentation]    Stop karaf on all controllers
-    ClusterManagement.Stop Single Member    1    msg=up: ODL1, ODL2, ODL3, down=none
-    ClusterManagement.Stop Single Member    2    msg=up: ODL2, ODL3, down=ODL1
-    ClusterManagement.Stop Single Member    3    msg=up: ODL3, down=ODL1, ODL2
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
+    ClusterManagement.Stop_Members_From_List_Or_All
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Bring Up All Instances
     [Documentation]    Bring up all controllers. Do not check for cluster sync until all nodes are
     ...    up. akka will not let nodes join until they are all back up if two were down.
-    ClusterManagement.Start Single Member    1    msg=up: none, down: ODL1, ODL2, ODL3    wait_for_sync=False
-    ClusterManagement.Start Single Member    2    msg=up: ~ODL1, down: ODL2, ODL3    wait_for_sync=False
-    ClusterManagement.Start Single Member    3    msg=up: ~ODL1, ~ODL2, down: ODL3
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
+    ClusterManagement.Start Members From List Or All
+    BuiltIn.Run Keyword And Ignore Error    ClusterManagement.Get Raft State Of Shard Of All Member Nodes    shard_name=default    shard_type=config
     [Teardown]    OpenStackOperations.Get Test Teardown Debugs    fail=False
 
 Connectivity Tests From Vm Instance2 In net_2 after recovering all nodes

diff --git a/csit/testplans/netvirt-3node-openstack.txt b/csit/testplans/netvirt-3node-openstack.txt
index 1eb224d0be09a4f0b267c8520050e7669dfcaf34..3b99f7997069635389d47cfe27049e33cbcc145c 100644 (file)
--- a/csit/testplans/netvirt-3node-openstack.txt
+++ b/csit/testplans/netvirt-3node-openstack.txt
@@ -8,4 +8,4 @@ integration/test/csit/suites/openstack/clustering/ha_l3.robot
 # Disable this test until https://bugzilla.redhat.com/show_bug.cgi?id=1488907 gets resolved and then reflect changes in healthcheck in haproxy
 # integration/test/csit/suites/openstack/clustering/ha_l3_block_port.robot
 integration/test/csit/suites/netvirt/vpnservice/vpn_basic.robot
-integration/test/csit/suites/netvirt/elan/elan.robot
+integration/test/csit/suites/netvirt/elan/elan.robot
\ No newline at end of file