Bug 8629: Do not remove replica in listener suites

Current Carbon implementation of cluster-wide listener
relies on local replica being available.
When the replica is shut down (or otherwise deactivated),
listener stops getting data change notifications.

This makes the suite perform explicit leader movement,
so every member still has its local replica.
As we can now control which member becomes new leader,
there is one more test case.

Also, the suite was renamed to better match what happens during test.

Change-Id: I3cccd2581e6a4612badd7b53a600da2e1d0f1794
Signed-off-by: Vratko Polak <vrpolak@cisco.com>

diff --git a/csit/libraries/controller/DdbCommons.robot b/csit/libraries/controller/DdbCommons.robot
index 6a626e8225e1df0892b69f8d51ce86455954e49c..88a9fe3fe8cff2dea29eb254c3b6094d16862d85 100644 (file)
--- a/csit/libraries/controller/DdbCommons.robot
+++ b/csit/libraries/controller/DdbCommons.robot
@@ -298,57 +298,53 @@ Get_Seconds_To_Time
     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 
     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 

-Remote_Listener_Test_Templ
-    [Arguments]    ${listener_node_role}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
-    [Documentation]    Implements remote listener test scenario.
+Listener_Stability_Test_Templ
+    [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
+    [Documentation]    Implements listener stability test scenario for module-based shards.

     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     ${all_indices} =    ClusterManagement.List_All_Indices
     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     ${all_indices} =    ClusterManagement.List_All_Indices

-    ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
-    ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
-    ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
-    MdsalLowlevel.Subscribe_Dtcl    ${listener_node_dst}
+    ${idx_from}    ${idx_to}    ${idx_listen} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}
+    ...    ${shard_type}
+    MdsalLowlevel.Subscribe_Dtcl    ${idx_listen}

     ${subscribed} =    BuiltIn.Set_Variable    ${True}
     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
     ${subscribed} =    BuiltIn.Set_Variable    ${True}
     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}

-    MdsalLowlevel.Shutdown_Shard_Replica    ${leader}    ${shard_name}
-    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${leader}    ${shard_name}    ${shard_type}
-    ${newleader}    ${newfollower_list} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}
-    ...    shard_type=${shard_type}    member_index_list=${follower_list}    verify_restconf=False
-    BuiltIn.Should_Not_Be_Equal_As_Numbers    ${leader}    ${newleader}
+    ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
+    ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}
+    ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
+    BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}

     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
     : FOR    ${resp}    IN    @{resp_list}
     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
     : FOR    ${resp}    IN    @{resp_list}
     \    TemplatedRequests.Check_Status_Code    @{resp}[2]

-    ${copy_matches} =    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}
+    ${copy_matches} =    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}

     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     BuiltIn.Should_Be_True    ${copy_matches}
     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     BuiltIn.Should_Be_True    ${copy_matches}

-    [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}
+    [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}

 
 

-Remote_Listener_PrefBasedShard_Test_Templ
-    [Arguments]    ${listener_node_role}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
-    [Documentation]    Implements listener isolation test scenario.
+Listener_Stability_PrefBasedShard_Test_Templ
+    [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${PREF_BASED_SHARD}    ${shard_type}=${SHARD_TYPE}
+    [Documentation]    Implements listener stability test scenario for prefix-based shards.

     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     ${all_indices} =    ClusterManagement.List_All_Indices
     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     ${all_indices} =    ClusterManagement.List_All_Indices

-    ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!    shard_type=${shard_type}    member_index_list=${all_indices}    verify_restconf=False
-    ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
-    ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
-    MdsalLowlevel.Subscribe_Ddtl    ${listener_node_dst}
+    ${idx_from}    ${idx_to}    ${idx_listen} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}    ${shard_name}!!
+    ...    ${shard_type}
+    MdsalLowlevel.Subscribe_Ddtl    ${idx_listen}

     ${subscribed} =    BuiltIn.Set_Variable    ${True}
     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}
     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
     ${subscribed} =    BuiltIn.Set_Variable    ${True}
     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${all_ip_list}    ${all_indices}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}
     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}

-    MdsalLowlevel.Shutdown_Prefix_Shard_Replica    ${leader}    ${shard_name}
-    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${leader}    ${shard_name}!!    ${shard_type}
-    ${newleader}    ${newfollower_list} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}!!
-    ...    member_index_list=${follower_list}    verify_restconf=False    shard_type=${shard_type}
-    BuiltIn.Should_Not_Be_Equal_As_Numbers    ${leader}    ${newleader}
+    MdsalLowlevel.Become_Prefix_Leader    ${idx_to}    ${shard_name}
+    ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!
+    ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
+    BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}

     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
     : FOR    ${resp}    IN    @{resp_list}
     \    TemplatedRequests.Check_Status_Code    @{resp}[2]
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
     : FOR    ${resp}    IN    @{resp_list}
     \    TemplatedRequests.Check_Status_Code    @{resp}[2]

-    ${copy_matches} =    MdsalLowlevel.Unsubscribe_Ddtl    ${listener_node_dst}
+    ${copy_matches} =    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}

     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     BuiltIn.Should_Be_True    ${copy_matches}
     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     BuiltIn.Should_Be_True    ${copy_matches}

-    [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Ddtl    ${listener_node_dst}
+    [Teardown]    BuiltIn.Run_Keyword_If    ${subscribed}    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}

 
 Create_Prefix_Based_Shard_And_Verify
     [Arguments]    ${prefix}=${PREF_BASED_SHARD}
 
 Create_Prefix_Based_Shard_And_Verify
     [Arguments]    ${prefix}=${PREF_BASED_SHARD}

diff --git a/csit/suites/controller/dom_data_broker/remote_listener.robot b/csit/suites/controller/dom_data_broker/listener_stability.robot
similarity index 64%
rename from csit/suites/controller/dom_data_broker/remote_listener.robot
rename to csit/suites/controller/dom_data_broker/listener_stability.robot
index 257ac33594dd7af494030fa05b8d0824f60a74da..1188984318352364e26ca89a8421f184452b4758 100644 (file)
--- a/csit/suites/controller/dom_data_broker/remote_listener.robot
+++ b/csit/suites/controller/dom_data_broker/listener_stability.robot
@@ -1,5 +1,5 @@
 *** Settings ***
 *** Settings ***

-Documentation     DOMDataBroker testing: Remote Listener
+Documentation     DOMDataBroker testing: Listener Stability for module-based shards

 ...
 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
 ...
 ...
 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
 ...


@@ -15,21 +15,30 @@ Suite Teardown    SSHLibrary.Close_All_Connections
 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 Default Tags      critical
 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
 Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 Default Tags      critical

-Test Template     DdbCommons.Remote_Listener_Test_Templ
+Test Template     DdbCommons.Listener_Stability_Test_Templ

 Library           SSHLibrary
 Resource          ${CURDIR}/../../../libraries/controller/DdbCommons.robot
 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
 
 *** Test Cases ***
 Library           SSHLibrary
 Resource          ${CURDIR}/../../../libraries/controller/DdbCommons.robot
 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
 
 *** Test Cases ***

-Listener_On_Shard_Leader_Node
-    [Documentation]    Listener runs on leader node when leader changed.
-    leader
+Move_Leader_From_Listener_Local_To_Remote
+    [Documentation]    Listener runs on leader node when leader is moved to remote node.
+    local    remote

 
 

-Restart
-    [Documentation]    Restart odl
+Restart_1
+    [Documentation]    Restart odl.

     [Template]
     DdbCommons.Restart_Test_Templ
 
     [Template]
     DdbCommons.Restart_Test_Templ
 

-Listener_On_Shard_Non_Leader_Node
-    [Documentation]    Listener runs on non-leader node when leader changed.
-    non-leader
+Move_Leader_From_Listener_Remote_To_Other_Remote
+    [Documentation]    Listener runs on follower node when leader is moved to the third node.
+    remote    remote
+
+Restart_2
+    [Documentation]    Restart odl.
+    [Template]
+    DdbCommons.Restart_Test_Templ
+
+Move_Leader_From_Listener_Remote_To_Local
+    [Documentation]    Listener runs on follower node when leader is moved to local node.
+    remote    local

diff --git a/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot b/csit/suites/controller/dom_data_broker/listener_stability_prefbasedshard.robot
similarity index 66%
rename from csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot
rename to csit/suites/controller/dom_data_broker/listener_stability_prefbasedshard.robot
index a7b17a9eb6c30152cea49e9cd91c4b93eac7b59a..46e6ba3dde3de6205b9e6eafd156ab19917de178 100644 (file)
--- a/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot
+++ b/csit/suites/controller/dom_data_broker/listener_stability_prefbasedshard.robot
@@ -1,5 +1,5 @@
 *** Settings ***
 *** Settings ***

-Documentation     DOMDataBroker testing: Remote Listener
+Documentation     DOMDataBroker testing: Listener Stability for prefix-based shards

 ...
 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
 ...
 ...
 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
 ...


@@ -17,23 +17,34 @@ Test Setup        BuiltIn.Run_Keywords    SetupUtils.Setup_Test_With_Logging_And
 Test Teardown     BuiltIn.Run_Keywords    DdbCommons.Remove_Prefix_Based_Shard_And_Verify
 ...               AND    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 Default Tags      critical
 Test Teardown     BuiltIn.Run_Keywords    DdbCommons.Remove_Prefix_Based_Shard_And_Verify
 ...               AND    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 Default Tags      critical

-Test Template     DdbCommons.Remote_Listener_PrefBasedShard_Test_Templ
+Test Template     DdbCommons.Listener_Stability_PrefBasedShard_Test_Templ

 Library           SSHLibrary
 Resource          ${CURDIR}/../../../libraries/controller/DdbCommons.robot
 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
 
 *** Test Cases ***
 Library           SSHLibrary
 Resource          ${CURDIR}/../../../libraries/controller/DdbCommons.robot
 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
 
 *** Test Cases ***

-Listener_On_Shard_Leader_Node
-    [Documentation]    Listener runs on leader node when leader changed.
-    leader
+Move_Leader_From_Listener_Local_To_Remote
+    [Documentation]    Listener runs on leader node when leader is moved to remote node.
+    local    remote

 
 

-Restart
+Restart_1

     [Documentation]    Restart odl.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     [Template]
     DdbCommons.Restart_Test_Templ
     [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
     [Documentation]    Restart odl.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     [Template]
     DdbCommons.Restart_Test_Templ
     [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 

-Listener_On_Shard_Non_Leader_Node
-    [Documentation]    Listener runs on non-leader node when leader changed.
-    non-leader
+Move_Leader_From_Listener_Remote_To_Other_Remote
+    [Documentation]    Listener runs on follower node when leader is moved to the third node.
+    remote    remote
+
+Restart_2
+    [Documentation]    Restart odl.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+    [Template]
+    DdbCommons.Restart_Test_Templ
+    [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+
+Move_Leader_From_Listener_Remote_To_Local
+    [Documentation]    Listener runs on follower node when leader is moved to local node.
+    remote    local

diff --git a/csit/testplans/controller-clustering.txt b/csit/testplans/controller-clustering.txt
index fe4811f7b607d7f062c681399383697f3c3faa5c..58841622e4dbfcd4940e70ffeb4f8ca92d42d012 100644 (file)
--- a/csit/testplans/controller-clustering.txt
+++ b/csit/testplans/controller-clustering.txt
@@ -34,9 +34,9 @@ integration/test/csit/suites/controller/dom_data_broker/client_isolation.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/dom_data_broker/client_isolation_prefbasedshard.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot

-integration/test/csit/suites/controller/dom_data_broker/remote_listener.robot
+integration/test/csit/suites/controller/dom_data_broker/listener_stability.robot

 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot

-integration/test/csit/suites/controller/dom_data_broker/remote_listener_prefbasedshard.robot
+integration/test/csit/suites/controller/dom_data_broker/listener_stability_prefbasedshard.robot

 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/cluster_singleton/master_stability.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot
 integration/test/csit/suites/controller/cluster_singleton/master_stability.robot
 integration/test/csit/suites/controller/dom_data_broker/restart_odl_with_tell_based_true.robot