From 4135aa11c0d0364412cbeb04e9be54eaa891207b Mon Sep 17 00:00:00 2001 From: Peter Gubka Date: Mon, 27 Feb 2017 11:58:50 +0100 Subject: [PATCH] Add Cluster Singleton Test Suites Change-Id: Ic836f1678570503f07580a53e53a408aeb2c5745 Signed-off-by: Peter Gubka --- csit/libraries/ClusterManagement.robot | 14 ++ csit/libraries/MdsalLowlevel.robot | 34 ++--- csit/libraries/controller/CsCommon.robot | 120 ++++++++++++++++++ .../chasing_the_leader.robot | 51 ++++++++ .../cluster_singleton/master_stability.robot | 66 ++++++++++ .../partition_and_heal.robot | 70 ++++++++++ csit/testplans/controller-clustering.txt | 3 + .../get_singleton_constant/post_data.xml | 0 .../register_flapping_singleton/post_data.xml | 0 .../register_singleton_constant/post_data.xml | 2 +- .../post_data.xml | 0 .../post_data.xml | 0 12 files changed, 344 insertions(+), 16 deletions(-) create mode 100644 csit/libraries/controller/CsCommon.robot create mode 100644 csit/suites/controller/cluster_singleton/chasing_the_leader.robot create mode 100644 csit/suites/controller/cluster_singleton/master_stability.robot create mode 100644 csit/suites/controller/cluster_singleton/partition_and_heal.robot create mode 100644 csit/variables/mdsal/lowlevelrpc/get_singleton_constant/post_data.xml create mode 100644 csit/variables/mdsal/lowlevelrpc/register_flapping_singleton/post_data.xml create mode 100644 csit/variables/mdsal/lowlevelrpc/unregister_flapping_singleton/post_data.xml create mode 100644 csit/variables/mdsal/lowlevelrpc/unregister_singleton_constant/post_data.xml diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot index 037f143a15..f6490eaf37 100644 --- a/csit/libraries/ClusterManagement.robot +++ b/csit/libraries/ClusterManagement.robot @@ -261,6 +261,20 @@ Get_Owner_And_Candidates_For_Device BuiltIn.Run_Keyword_If '${status}'=='FAIL' BuiltIn.Fail Could not parse owner and candidates for device ${device_name} [Return] @{results} +Check_Old_Owner_Stays_Elected_For_Device + [Arguments] ${device_name} ${device_type} ${old_owner} ${node_to_ask} + [Documentation] Verify the owner remain the same as ${old_owner} + ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device ${device_name} ${device_type} ${node_to_ask} + BuiltIn.Should_Be_Equal_As_numbers ${old_owner} ${owner} + BuiltIn.Return_From_Keyword ${owner} ${candidates} + +Check_New_Owner_Got_Elected_For_Device + [Arguments] ${device_name} ${device_type} ${old_owner} ${node_to_ask} + [Documentation] Verify new owner was elected comparing to ${old_owner} + ${owner} ${candidates} = Get_Owner_And_Candidates_For_Device ${device_name} ${device_type} ${node_to_ask} + BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_owner} ${owner} + BuiltIn.Return_From_Keyword ${owner} ${candidates} + Get_Owner_And_Candidates_For_Type_And_Id [Arguments] ${type} ${id} ${member_index} ${require_candidate_list}=${EMPTY} [Documentation] Returns the owner and a list of candidates for entity specified by ${type} and ${id} diff --git a/csit/libraries/MdsalLowlevel.robot b/csit/libraries/MdsalLowlevel.robot index dcb82c50d4..603371e7cb 100644 --- a/csit/libraries/MdsalLowlevel.robot +++ b/csit/libraries/MdsalLowlevel.robot @@ -50,11 +50,12 @@ ${WRITE_TRANSACTIONS_DIR} ${RPC_DIR}/write_transactions *** Keywords *** Get_Constant [Arguments] ${member_index} ${explicit_status_codes}=${NO_STATUS_CODES} - [Documentation] Invoke get-constant rpc on the requested member and return the registered constant. The ${explicit_status_codes} is a list - ... of http status codes for which the rpc call is considered as passed and is used for calls with expected failures on odl's side, such as - ... calling the rpc on isolated node etc. + [Documentation] Invoke get-constant rpc on the requested member and return the registered constant unless explicit status code is expected. + ... The ${explicit_status_codes} is a list of http status codes for which the rpc call is considered as passed and is used for calls with + ... expected failures on odl's side, such as calling the rpc on isolated node etc. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} ${text} = TemplatedRequests.Post_As_Xml_Templated ${GET_CONSTANT_DIR} session=${session} explicit_status_codes=${explicit_status_codes} + BuiltIn.Return_From_Keyword_If """${explicit_status_codes}""" != """${NO_STATUS_CODES}""" ${xml} = XML.Parse_Xml ${text} ${constant} = XML.Get_Element_Text ${xml} xpath=constant BuiltIn.Return_From_Keyword ${constant} @@ -72,13 +73,16 @@ Get_Contexted_Constant BuiltIn.Return_From_Keyword ${constant} Get_Singleton_Constant - [Arguments] ${member_index} - [Documentation] TODO: more desctiptive comment than: Invoke get-singleton-constant rpc. + [Arguments] ${member_index} ${explicit_status_codes}=${NO_STATUS_CODES} + [Documentation] Invoke get-singleton-constant rpc on the requested member and return the registered constant unless explicit status code is + ... expected. The ${explicit_status_codes} is a list of http status codes for which the rpc call is considered as passed and is used for calls + ... with expected failures on odl's side, such as calling the rpc on isolated node etc. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} - ${uri} = TemplatedRequests.Resolve_Text_From_Template_Folder folder=${GET_SINGLETON_CONSTANT_DIR} base_name=location extension=uri - ${text} = TemplatedRequests.Post_To_Uri uri=${uri} data=${EMPTY} accept=${ACCEPT_JSON} content_type=${HEADERS_YANG_JSON} session=${session} - BuiltIn.Fail TODO: to format output data - BuiltIn.Return_From_Keyword ${formatted_output} + ${text} = TemplatedRequests.Post_As_Xml_Templated ${GET_SINGLETON_CONSTANT_DIR} session=${session} explicit_status_codes=${explicit_status_codes} + BuiltIn.Return_From_Keyword_If """${explicit_status_codes}""" != """${NO_STATUS_CODES}""" + ${xml} = XML.Parse_Xml ${text} + ${constant} = XML.Get_Element_Text ${xml} xpath=constant + BuiltIn.Return_From_Keyword ${constant} Register_Constant [Arguments] ${member_index} ${constant} @@ -104,22 +108,22 @@ Unregister_Singleton_Constant [Arguments] ${member_index} [Documentation] TODO: more desctiptive comment than: Invoke unregister-singleton-constant rpc. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} - ${uri} = TemplatedRequests.Resolve_Text_From_Template_Folder folder=${UNREGISTER_SINGLETON_CONSTANT_DIR} base_name=location extension=uri - ${text} = TemplatedRequests.Post_To_Uri uri=${uri} data=${EMPTY} accept=${ACCEPT_JSON} content_type=${HEADERS_YANG_JSON} session=${session} + TemplatedRequests.Post_As_Xml_Templated ${UNREGISTER_SINGLETON_CONSTANT_DIR} session=${session} Register_Flapping_Singleton [Arguments] ${member_index} [Documentation] TODO: more desctiptive comment than: Invoke register-flapping-singleton rpc. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} - ${uri} = TemplatedRequests.Resolve_Text_From_Template_Folder folder=${REGISTER_FLAPPING_SINGLETON_DIR} base_name=location extension=uri - TemplatedRequests.Post_To_Uri uri=${uri} data=${EMPTY} accept=${ACCEPT_JSON} content_type=${HEADERS_YANG_JSON} session=${session} + TemplatedRequests.Post_As_Xml_Templated ${REGISTER_FLAPPING_SINGLETON_DIR} session=${session} Unregister_Flapping_Singleton [Arguments] ${member_index} [Documentation] TODO: more desctiptive comment than: Invoke unregister-flapping-singleton rpc. ${session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${member_index} - ${uri} = TemplatedRequests.Resolve_Text_From_Template_Folder folder=${UNREGISTER_FLAPPING_SINGLETON_DIR} base_name=location extension=uri - ${text} = TemplatedRequests.Post_To_Uri uri=${uri} data=${EMPTY} accept=${ACCEPT_JSON} content_type=${HEADERS_YANG_JSON} session=${session} + ${text} = TemplatedRequests.Post_As_Xml_Templated ${UNREGISTER_FLAPPING_SINGLETON_DIR} session=${session} + ${xml} = XML.Parse_Xml ${text} + ${count} = XML.Get_Element_Text ${xml} xpath=flap-count + BuiltIn.Return_From_Keyword ${count} Write_Transactions [Arguments] ${member_index} ${seconds} ${trans_per_sec} ${chained_trans}=${True} diff --git a/csit/libraries/controller/CsCommon.robot b/csit/libraries/controller/CsCommon.robot new file mode 100644 index 0000000000..8e31f05fbb --- /dev/null +++ b/csit/libraries/controller/CsCommon.robot @@ -0,0 +1,120 @@ +*** Settings *** +Documentation Cluster Singleton testing: Common Keywords +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... Cluster Singleton service is designed to ensure that only one instance of an +... application is registered globally in the cluster. +... +... Creates and uses the following suite variables: +... Created by Cluster_Singleton_Init: +... ${all_indices} +... ${exp_candidates} +... Created by Get_And_Save_Present_CsOwner_And_CsCandidates: +... ${cs_owner} +... ${cs_candidates} +... Created by Isolate_Owner_And_Verify_Isolated +... ${cs_isolated_index} +Library Collections +Resource ${CURDIR}/../ClusterManagement.robot +Resource ${CURDIR}/../MdsalLowlevel.robot +Resource ${CURDIR}/../WaitForFailure.robot + +*** Variables *** +${CS_DEVICE_NAME} get-singleton-constant-service'] +${CS_DEVICE_TYPE} org.opendaylight.mdsal.ServiceEntityType +${CS_CONSTANT_PREFIX} constant- +@{CS_STATUS_ISOLATED} ${501} + +*** Keywords *** +Cluster_Singleton_Init + [Documentation] Resouce initial keyword. Creates {exp_candidates} and {all_indices} suite variables which are + ... used in other keywords. + ${exp_candidates} = BuiltIn.Create_List + BuiltIn.Set_Suite_Variable ${exp_candidates} + ${all_indices} = ClusterManagement.List_All_Indices + BuiltIn.Set_Suite_Variable ${all_indices} + +Register_Singleton_And_Update_Expected_Candidates + [Arguments] ${member_index} ${constant} + [Documentation] Register the singleton candidate and add it to the list of ${exp_candidates}. + MdsalLowlevel.Register_Singleton_Constant ${member_index} ${constant} + Collections.Append_To_List ${exp_candidates} ${member_index} + Collections.Sort_List ${exp_candidates} + +Unregister_Singleton_And_Update_Expected_Candidates + [Arguments] ${member_index} + [Documentation] Unregister the singleton candidate. Also remove it from the list of ${exp_candidates}. + MdsalLowlevel.Unregister_Singleton_Constant ${member_index} + Collections.Remove_Values_From_List ${exp_candidates} ${member_index} + +Verify_Owner_And_Candidates_Stable + [Arguments] ${owner_index} + [Documentation] Fail if the actual owner is different from ${owner_index} or if the actual candidate list is different from ${exp_candidates}. + ${actual_owner} ${actual_candidates} ClusterManagement.Check_Old_Owner_Stays_Elected_For_Device ${CS_DEVICE_NAME} ${CS_DEVICE_TYPE} ${owner_index} ${owner_index} + Collections.Lists_Should_Be_Equal ${exp_candidates} ${actual_candidates} + +Monitor_Owner_And_Candidates_Stability + [Arguments] ${monitoring_duration} ${owner_index} + [Documentation] Verify that the owner remains on the same node after non-owner candidate is unregistered. + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${monitoring_duration} 3s Verify_Owner_And_Candidates_Stable ${owner_index} + +Register_Singleton_Constant_On_Nodes + [Arguments] ${index_list} + [Documentation] Register a candidate application on given nodes. + : FOR ${index} IN @{index_list} + \ Register_Singleton_And_Update_Expected_Candidates ${index} ${CS_CONSTANT_PREFIX}${index} + +Unregister_Singleton_Constant_On_Nodes + [Arguments] ${index_list} + [Documentation] Unregister the application from given nodes. + : FOR ${index} IN @{index_list} + \ Unregister_Singleton_And_Update_Expected_Candidates ${index} + +Get_And_Save_Present_CsOwner_And_CsCandidates + [Arguments] ${node_to_ask} + [Documentation] Store owner index into suite variables. + ${cs_owner} ${cs_candidates} ClusterManagement.Get_Owner_And_Candidates_For_Device ${CS_DEVICE_NAME} ${CS_DEVICE_TYPE} ${node_to_ask} + BuiltIn.Set_Suite_Variable ${cs_owner} + BuiltIn.Set_Suite_Variable ${cs_candidates} + BuiltIn.Return_From_Keyword ${cs_owner} ${cs_candidates} + +Verify_Singleton_Constant_On_Node + [Arguments] ${node_to_ask} ${exp_constant} + [Documentation] Verify that the expected constant is return from the given node. + ${constant} = MdsalLowlevel.Get_Singleton_Constant ${node_to_ask} + BuiltIn.Should_Be_Equal ${exp_constant} ${constant} + +Verify_Singleton_Constant_On_Nodes + [Arguments] ${index_list} ${exp_constant} + [Documentation] Iterate over all cluster nodes and all should return expected constant. + : FOR ${index} IN @{index_list} + \ Verify_Singleton_Constant_On_Node ${index} ${exp_constant} + +Verify_Singleton_Constant_During_Isolation + [Documentation] Iterate over all cluster nodes. Isolated node should return http status code ${CS_STATUS_ISOLATED}. All non-isolated nodes should + ... return correct constant. + : FOR ${index} IN @{all_indices} + \ BuiltIn.Run_Keyword_If "${index}" == "${cs_isolated_index}" MdsalLowlevel.Get_Singleton_Constant ${index} explicit_status_codes=${CS_STATUS_ISOLATED} + \ BuiltIn.Run_Keyword_Unless "${index}" == "${cs_isolated_index}" Verify_Singleton_Constant_On_Node ${index} ${CS_CONSTANT_PREFIX}${cs_owner} + +Isolate_Owner_And_Verify_Isolated + [Documentation] Isolate the owner cluster node. Wait until the new owner is elected and store new values of owner and candidates. Then wait + ... for isolated node to respond correctly when isolated. + ClusterManagement.Isolate_Member_From_List_Or_All ${cs_owner} + BuiltIn.Set_Suite_Variable ${cs_isolated_index} ${cs_owner} + ${non_isolated_list} = ClusterManagement.List_Indices_Minus_Member ${cs_isolated_index} member_index_list=${all_indices} + ${node_to_ask} = Collections.Get_From_list ${non_isolated_list} 0 + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Check_New_Owner_Got_Elected_For_Device ${CS_DEVICE_NAME} ${CS_DEVICE_TYPE} ${cs_isolated_index} + ... ${node_to_ask} + Get_And_Save_Present_CsOwner_And_CsCandidates ${node_to_ask} + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s MdsalLowlevel.Get_Singleton_Constant ${cs_isolated_index} explicit_status_codes=${CS_STATUS_ISOLATED} + +Rejoin_Node_And_Verify_Rejoined + [Documentation] Rejoin isolated node. + ClusterManagement.Rejoin_Member_From_List_Or_All ${cs_isolated_index} + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s Verify_Singleton_Constant_On_Node ${cs_isolated_index} ${CS_CONSTANT_PREFIX}${cs_owner} diff --git a/csit/suites/controller/cluster_singleton/chasing_the_leader.robot b/csit/suites/controller/cluster_singleton/chasing_the_leader.robot new file mode 100644 index 0000000000..7feeef02bb --- /dev/null +++ b/csit/suites/controller/cluster_singleton/chasing_the_leader.robot @@ -0,0 +1,51 @@ +*** Settings *** +Documentation Cluster Singleton testing: Chasing the Leader +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... This test aims to establish the service operates correctly when faced with +... rapid application transitions without having a stabilized application. +Suite Setup SetupUtils.Setup_Utils_For_Setup_And_Teardown +Suite Teardown SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Library Collections +Library DateTime +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/MdsalLowlevel.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot + +*** Variables *** +${TEST_DURATION} 1m +${ACCEPTED_PER_SEC_RATE} 100 + +*** Test Cases *** +Register_Candidates + [Documentation] Register a candidate application on each node which starts the test. + ${index_list} = ClusterManagement.List_All_Indices + : FOR ${index} IN @{index_list} + \ MdsalLowlevel.Register_Flapping_Singleton ${index} + +Do_Nothing + [Documentation] Do nothing for the time of the test duration, because there is no api to monitor the statistics + ... during the test execution. Statistics are available only at the end, when unregister-flapping-singleton rpc is + ... called. + BuiltIn.Sleep ${TEST_DURATION} + +Unregister_Candidates_And_Validate_Criteria + [Documentation] Unregister the testing service and check recevied statistics. + ${index_list} = ClusterManagement.List_All_Indices + ${movements_count} = BuiltIn.Set_Variable ${0} + : FOR ${index} IN @{index_list} + \ ${count} = MdsalLowlevel.Unregister_Flapping_Singleton ${index} + \ BuiltIn.Run_Keyword_If ${count} < 0 BuiltIn.Fail No failure should have occured during the ${TEST_DURATION} timeout. + \ ${movements_count} = BuiltIn.Evaluate ${movements_count}+${count} + ${seconds} = DateTime.Convert_Time ${TEST_DURATION} + ${rate} = BuiltIn.Evaluate ${movements_count}/${seconds} + BuiltIn.Run_Keyword_If ${rate} < ${ACCEPTED_PER_SEC_RATE} BuiltIn.Fail Acceptance rate ${ACCEPTED_PER_SEC_RATE} not reached, actual rate is ${rate}. diff --git a/csit/suites/controller/cluster_singleton/master_stability.robot b/csit/suites/controller/cluster_singleton/master_stability.robot new file mode 100644 index 0000000000..cd7eca830f --- /dev/null +++ b/csit/suites/controller/cluster_singleton/master_stability.robot @@ -0,0 +1,66 @@ +*** Settings *** +Documentation Cluster Singleton testing: Master Stability +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... Cluster Singleton service is designed to ensure that only one instance of an +... application is registered globally in the cluster. +... The goal is to establish the service operates correctly in face of application +... registration changing without moving the active instance. +Suite Setup Setup_Keyword +Suite Teardown SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Library Collections +Library SSHLibrary +Resource ${CURDIR}/../../../libraries/controller/CsCommon.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot +Resource ${CURDIR}/../../../libraries/WaitForFailure.robot + +*** Variables *** +${STABILITY_TIMEOUT} 1m + +*** Test Cases *** +Register_Singleton_Constant_On_Each_Node_And_Verify + [Documentation] Register a candidate application on each node. + CsCommon.Register_Singleton_Constant_On_Nodes ${all_indices} + ${owner} ${candidates}= CsCommon.Get_And_Save_Present_CsOwner_And_CsCandidates 1 + BuiltIn.Wait_Until_Keyword_Succeeds 3x 2s CsCommon.Verify_Singleton_Constant_On_Nodes ${all_indices} ${CS_CONSTANT_PREFIX}${owner} + +Unregister_Singleton_Constant_On_Non_Master_Node + [Documentation] Unregister the application on a non master node. + ${unregistered_node} = Get_Node_Idx_To_Unregister + CsCommon.Unregister_Singleton_And_Update_Expected_Candidates ${unregistered_node} + BuiltIn.Set_Suite_Variable ${unregistered_node} + +Monitor_Stability_While_Unregistered + [Documentation] Verify that the owner remains on the same node after non-owner candidate is unregistered. + CsCommon.Monitor_Owner_And_Candidates_Stability ${STABILITY_TIMEOUT} ${cs_owner} + +Reregister_Singleton_Constant + [Documentation] Re-register the unregistered candidate. + CsCommon.Register_Singleton_And_Update_Expected_Candidates ${unregistered_node} ${CS_CONSTANT_PREFIX}${unregistered_node} + +Verify_Stability_After_Reregistration + [Documentation] Verify that the owner remains on the same node after the unregistered candidate re-registration. + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout 5x 3s CsCommon.Verify_Owner_And_Candidates_Stable ${cs_owner} + +Unregister_Singleton_Constant_On_Each_Node + [Documentation] Unregister the application from each node. + CsCommon.Unregister_Singleton_Constant_On_Nodes ${all_indices} + +*** Keywords *** +Setup_Keyword + SetupUtils.Setup_Utils_For_Setup_And_Teardown + CsCommon.Cluster_Singleton_Init + +Get_Node_Idx_To_Unregister + [Documentation] Return the first non owner node from the stored candidate list. + : FOR ${index} IN @{cs_candidates} + \ BuiltIn.Return_From_Keyword_If "${index}" != "${cs_owner}" ${index} diff --git a/csit/suites/controller/cluster_singleton/partition_and_heal.robot b/csit/suites/controller/cluster_singleton/partition_and_heal.robot new file mode 100644 index 0000000000..ddfb8fe016 --- /dev/null +++ b/csit/suites/controller/cluster_singleton/partition_and_heal.robot @@ -0,0 +1,70 @@ +*** Settings *** +Documentation Cluster Singleton testing: Partition And Heal +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... Cluster Singleton service is designed to ensure that only one instance of +... an application is registered globally in the cluster. +... The goal is to establish the service operates correctly in face of node +... failures. +Suite Setup Setup_Keyword +Suite Teardown SSHLibrary.Close_All_Connections +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Default Tags critical +Library Collections +Library SSHLibrary +Library RequestsLibrary +Resource ${CURDIR}/../../../libraries/controller/CsCommon.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/MdsalLowlevel.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot +Resource ${CURDIR}/../../../libraries/WaitForFailure.robot + +*** Variables *** +${STABILITY_TIMEOUT_ISOLATED} 120s +${STABILITY_TIMEOUT_REJOINED} 60s +@{STATUS_ISOLATED} ${501} + +*** Test Cases *** +Register_Singleton_Constant_On_Each_Node + [Documentation] Register a candidate application on each node. + CsCommon.Register_Singleton_Constant_On_Nodes ${all_indices} + +Verify_Singleton_Constant_On_Each_Node + [Documentation] Store the owner and candidates of the application and initially verify that all + ... odl nodes are used. + ${owner} ${candidates}= CsCommon.Get_And_Save_Present_CsOwner_And_CsCandidates 1 + BuiltIn.Wait_Until_Keyword_Succeeds 3x 2s CsCommon.Verify_Singleton_Constant_On_Nodes ${all_indices} ${CS_CONSTANT_PREFIX}${owner} + +Isolate_Owner_Node + [Documentation] Isolate the cluster node which is the owner. Wait until the new owner is elected and store + ... new values of owner and candidates. + CsCommon.Isolate_Owner_And_Verify_Isolated + +Monitor_Stability_While_Isolated + [Documentation] Monitor the stability of the singleton application and fail the the owner is changed during the monitoring. + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${STABILITY_TIMEOUT_ISOLATED} 3s CsCommon.Verify_Singleton_Constant_During_Isolation + +Rejoin_Isolated_node + [Documentation] Rejoin isolated node. + [Tags] @{NO_TAGS} + CsCommon.Rejoin_Node_And_Verify_Rejoined + +Verify_Stability_After_Rejoin + [Documentation] Verify that the rejoining of the isolated node does not change the singleton owner. + WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout ${STABILITY_TIMEOUT_REJOINED} 3s CsCommon.Verify_Singleton_Constant_On_Nodes ${all_indices} ${CS_CONSTANT_PREFIX}${cs_owner} + +Unregister_Singleton_Constant_On_Each_Node + [Documentation] Unregister the application on every node. + CsCommon.Unregister_Singleton_Constant_On_Nodes ${all_indices} + +*** Keywords *** +Setup_Keyword + [Documentation] Suite setup. + SetupUtils.Setup_Utils_For_Setup_And_Teardown + CsCommon.Cluster_Singleton_Init diff --git a/csit/testplans/controller-clustering.txt b/csit/testplans/controller-clustering.txt index d958357bc0..6edf4e19b8 100644 --- a/csit/testplans/controller-clustering.txt +++ b/csit/testplans/controller-clustering.txt @@ -10,6 +10,9 @@ integration/test/csit/suites/controller/singleton_service/global_rpc_freeze.robo integration/test/csit/suites/controller/singleton_service/global_rpc_isolate.robot integration/test/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot integration/test/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot +integration/test/csit/suites/controller/cluster_singleton/master_stability.robot +integration/test/csit/suites/controller/cluster_singleton/partition_and_heal.robot +integration/test/csit/suites/controller/cluster_singleton/chasing_the_leader.robot integration/test/csit/suites/controller/dom_rpc_broker/rpc_provider_precedence.robot integration/test/csit/suites/controller/dom_rpc_broker/rpc_provider_partition_and_heal.robot integration/test/csit/suites/controller/dom_rpc_broker/action_provider_precedence.robot diff --git a/csit/variables/mdsal/lowlevelrpc/get_singleton_constant/post_data.xml b/csit/variables/mdsal/lowlevelrpc/get_singleton_constant/post_data.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/csit/variables/mdsal/lowlevelrpc/register_flapping_singleton/post_data.xml b/csit/variables/mdsal/lowlevelrpc/register_flapping_singleton/post_data.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/csit/variables/mdsal/lowlevelrpc/register_singleton_constant/post_data.xml b/csit/variables/mdsal/lowlevelrpc/register_singleton_constant/post_data.xml index 9987913a5e..01a7dabd52 100644 --- a/csit/variables/mdsal/lowlevelrpc/register_singleton_constant/post_data.xml +++ b/csit/variables/mdsal/lowlevelrpc/register_singleton_constant/post_data.xml @@ -1,3 +1,3 @@ - + $CONSTANT diff --git a/csit/variables/mdsal/lowlevelrpc/unregister_flapping_singleton/post_data.xml b/csit/variables/mdsal/lowlevelrpc/unregister_flapping_singleton/post_data.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/csit/variables/mdsal/lowlevelrpc/unregister_singleton_constant/post_data.xml b/csit/variables/mdsal/lowlevelrpc/unregister_singleton_constant/post_data.xml new file mode 100644 index 0000000000..e69de29bb2 -- 2.36.6