From 1450d8a59ff94e9f1252563cb19a863bc74c4919 Mon Sep 17 00:00:00 2001 From: Peter Gubka Date: Fri, 27 Jan 2017 16:53:39 +0100 Subject: [PATCH] Add isolation scenarios for clustered car/people tests Change-Id: Ie0e282df9e6a01aa8e0e3ef116f016d6378ac275 Signed-off-by: Peter Gubka --- csit/libraries/CarPeople.robot | 21 +++- csit/libraries/ClusterManagement.robot | 9 ++ .../buycar_failover_isolation.robot | 105 ++++++++++++++++ .../car_failover_crud_isolation.robot | 115 ++++++++++++++++++ csit/testplans/controller-clustering.txt | 2 + 5 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot create mode 100644 csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot diff --git a/csit/libraries/CarPeople.robot b/csit/libraries/CarPeople.robot index 3ff015d21c..62b7de5ff5 100644 --- a/csit/libraries/CarPeople.robot +++ b/csit/libraries/CarPeople.robot @@ -1,7 +1,7 @@ *** Settings *** Documentation Resource housing Keywords common to tests which interact with car/people models. ... -... Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +... Copyright (c) 2016-2017 Cisco Systems, Inc. and others. All rights reserved. ... ... This program and the accompanying materials are made available under the ... terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -60,3 +60,22 @@ Set_Variables_For_Shard BuiltIn.Set_Suite_Variable \${${shard_name}_follower_sessions} ${sessions} ${first_follower_session} = Collections.Get_From_List ${sessions} 0 BuiltIn.Set_Suite_Variable \${${shard_name}_first_follower_session} ${first_follower_session} + +Set_Tmp_Variables_For_Shard_For_Nodes + [Arguments] ${member_index_list} ${shard_name}=car ${shard_type}=config + [Documentation] Get current leader and followers for given shard. Can be used for less nodes than full odl configuration. + ... Variable names do not contain neither node nor shard names, so the variables are only suitable for temporary use, as indicated by Tmp in the keyword name. + ... This keyword sets the following suite variables: + ... ${new_leader_session} - http session for the leader node + ... ${new_follower_sessions} - list of http sessions for the follower nodes + ... ${new_first_follower_session} - http session for the first follower node + ${leader} ${follower_list} = ClusterManagement.Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${member_index_list} + ${leader_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${leader} + BuiltIn.Set_Suite_Variable \${new_leader_session} ${leader_session} + ${sessions} = BuiltIn.Create_List + : FOR ${follower_index} IN @{follower_list} + \ ${follower_session} = ClusterManagement.Resolve_Http_Session_For_Member member_index=${follower_index} + \ Collections.Append_To_List ${sessions} ${follower_session} + BuiltIn.Set_Suite_Variable \${new_follower_sessions} ${sessions} + ${first_follower_session} = Collections.Get_From_List ${sessions} 0 + BuiltIn.Set_Suite_Variable \${new_first_follower_session} ${first_follower_session} diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot index 6d1da90aba..267f3bef2c 100644 --- a/csit/libraries/ClusterManagement.robot +++ b/csit/libraries/ClusterManagement.robot @@ -140,6 +140,15 @@ Get_Raft_State_Of_Shard_At_Member ${raft_state} = Collections.Get_From_Dictionary ${value} RaftState [Return] ${raft_state} +Verify_Shard_Leader_Elected + [Arguments] ${shard_name} ${shard_type} ${new_elected} ${old_leader} ${member_index_list}=${EMPTY} + [Documentation] Verify new leader was elected or remained the same. Bool paramter ${new_elected} indicates if + ... new leader is elected or should remained the same as ${old_leader} + ${leader} ${followers}= Get_Leader_And_Followers_For_Shard shard_name=${shard_name} shard_type=${shard_type} member_index_list=${member_index_list} + BuiltIn.Run_Keyword_If ${new_elected} BuiltIn.Should_Not_Be_Equal_As_Numbers ${old_leader} ${leader} + BuiltIn.Run_Keyword_Unless ${new_elected} BuiltIn.Should_Be_Equal_As_numbers ${old_leader} ${leader} + BuiltIn.Return_From_Keyword ${leader} ${followers} + Verify_Owner_And_Successors_For_Device [Arguments] ${device_name} ${device_type} ${member_index} ${candidate_list}=${EMPTY} ${after_stop}=False [Documentation] Returns the owner and successors for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}. diff --git a/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot b/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot new file mode 100644 index 0000000000..ac2cf134dc --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot @@ -0,0 +1,105 @@ +*** Settings *** +Documentation This test focuses on testing buy-car RPC over the Leader isolation. +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... All purchases are against the same node, which is the first one to get isolated. +... +... All data is deleted at the end of the suite. +... This suite expects car, people and car-people modules to have separate Shards. +Suite Setup Setup +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CARPEOPLE_ITEMS} ${100} +${MEMBER_START_TIMEOUT} 300s +@{SHARD_NAME_LIST} car people car-people +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Add_Cars_To_Leader_And_Verify + [Documentation] Add all needed cars to car Leader, verify on each member. + ${car_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${car_items} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${car_items} + +Add_People_To_First_Follower_And_Verify + [Documentation] Add all needed people to people first Follower, verify on each member. + ${people_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 + CarPeople.Add_Several_People session=${people_first_follower_session} iterations=${people_items} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} verify=True iterations=${people_items} + +Buy_Cars_On_Leader_And_Verify + [Documentation] Buy some cars on the leader member. + ${iter_start} = BuiltIn.Evaluate 0 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 1 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Buy_Cars_On_Follower_And_Verify + [Documentation] Buy some cars on the first follower member. + ${iter_start} = BuiltIn.Evaluate 1 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${car-people_first_follower_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 2 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Isolate_and_Rejoin_People_Leader + [Documentation] Previous people Leader is isolated. We should never stop the people first follower, this is where people are registered. + ClusterManagement.Isolate_Member_From_List_Or_All ${people_leader_index} + BuiltIn.Wait_Until_Keyword_Succeeds 10x 2s ClusterManagement.Verify_Shard_Leader_Elected people config ${True} + ... ${people_leader_index} ${people_follower_indices} + ClusterManagement.Rejoin_Member_From_List_Or_All ${people_leader_index} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +Buy_Cars_On_Leader_After_Rejoin_And_Verify + [Documentation] Buy some cars on the leader member. + ${iter_start} = BuiltIn.Evaluate 2 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 3 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Buy_Cars_On_Follower_After_Rejoin_And_Verify + [Documentation] Buy some cars on the first follower member. + ${iter_start} = BuiltIn.Evaluate 3 * ${CARPEOPLE_ITEMS} + 1 + CarPeople.Buy_Several_Cars session=${car-people_first_follower_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} + ${total_iterations} = BuiltIn.Evaluate 4 * ${CARPEOPLE_ITEMS} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + +Delete_All_CarPeople + [Documentation] DELETE car-people container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/car-people session=${car-people_leader_session} + +Delete_All_People + [Documentation] DELETE people container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/people session=${people_leader_session} + +Delete_All_Cars + [Documentation] DELETE cars container. No verification beyond http status. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${car_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize shard leaders, compute item distribution. + SetupUtils.Setup_Utils_For_Setup_And_Teardown + Set_Variables_For_Shard shard_name=car + Set_Variables_For_Shard shard_name=people + Set_Variables_For_Shard shard_name=car-people + ${leader_list} = BuiltIn.Create_List ${car-people_leader_index} diff --git a/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot b/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot new file mode 100644 index 0000000000..fe0ca5cfac --- /dev/null +++ b/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot @@ -0,0 +1,115 @@ +*** Settings *** +Documentation Suite mixing basic operations with isolation of car Leader. +... +... Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... +... This test isolates the current leader of the "car" shard and then executes CRD +... operations on the new leader and a new follower. The isolated member is brought back. +... This suite uses 3 different car sets, same size but different starting ID. +... +... Other models and shards (people, car-people) are not accessed by this suite. +... +... All data is deleted at the end of the suite. +... This suite expects car module to have a separate Shard. +Suite Setup Setup +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Default Tags clustering carpeople critical +Library Collections +Resource ${CURDIR}/../../../libraries/CarPeople.robot +Resource ${CURDIR}/../../../libraries/ClusterManagement.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot +Resource ${CURDIR}/../../../libraries/TemplatedRequests.robot +Variables ${CURDIR}/../../../variables/Variables.py + +*** Variables *** +${CAR_ITEMS} 30 +${FOLLOWER_2NODE_START_I} 300 +${LEADER_2NODE_START_I} 200 +${MEMBER_START_TIMEOUT} 300s +${ORIGINAL_START_I} 100 +${SHARD_TYPE} config +${SHARD_NAME} car +@{SHARD_NAME_LIST} ${SHARD_NAME} +${VAR_DIR} ${CURDIR}/../../../variables/carpeople/crud + +*** Test Cases *** +Add_Original_Cars_On_Old_Leader_And_Verify + [Documentation] Add initial cars on car Leader. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + : FOR ${session} IN @{ClusterManagement__session_list} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + +Isolate_Original_Car_Leader + [Documentation] Isolate the car Leader to cause a new leader to get elected. + ClusterManagement.Isolate_Member_From_List_Or_All ${car_leader_index} + +Wait_For_New_Leader + [Documentation] Wait until new car Leader is elected. + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes member_index_list=${car_follower_indices} shard_name=${SHARD_NAME} shard_type=${SHARD_TYPE} + +See_Original_Cars_On_New_Leader + [Documentation] GET cars from new Leader, should be the initial ones. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + +See_Original_Cars_On_New_Followers + [Documentation] The same check on other existing member(s). + : FOR ${session} IN @{new_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + +Delete_Original_Cars_On_New_Leader + [Documentation] Delete cars on the new Leader. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${new_leader_session} + +Add_Leader_Cars_On_New_Leader + [Documentation] Add cars on the new Leader. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} iterations=${CAR_ITEMS} iter_start=${LEADER_2NODE_START_I} + +See_Leader_Cars_On_New_Leader + [Documentation] GET cars from new Leader, should be the new ones. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${LEADER_2NODE_START_I} + +See_Leader_Cars_On_New_Followers + [Documentation] The same check on other existing members. + : FOR ${session} IN @{new_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${LEADER_2NODE_START_I} + +Delete_Leader_Cars_On_New_First_Follower + [Documentation] Delete cars in new first Follower. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${new_first_follower_session} + +Add_Follower_Cars_On_New_First_Follower + [Documentation] Add cars on the new first Follower. + TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${new_first_follower_session} iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +See_Folower_Cars_On_New_Leader + [Documentation] Get cars from the new Leader, should be the ones added on follower. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${new_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +See_Follower_Cars_On_New_Followers + [Documentation] The same check on other existing members. + : FOR ${session} IN @{new_follower_sessions} + \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +Rejoin_Old_Car_Leader + [Documentation] Rejoin the isolated member without deleting the persisted data. + ClusterManagement.Rejoin_Member_From_List_Or_All ${car_leader_index} + BuiltIn.Wait_Until_Keyword_Succeeds 30s 2s ClusterManagement.Verify_Leader_Exists_For_Each_Shard shard_name_list=${SHARD_NAME_LIST} shard_type=config + +See_Folower_Cars_On_Old_Leader + [Documentation] GET cars from the restarted member, should be the ones added on follower. + TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} verify=True iterations=${CAR_ITEMS} iter_start=${FOLLOWER_2NODE_START_I} + +Delete_Follower_Cars_On_New_Leader + [Documentation] Delete cars on the last Leader. + TemplatedRequests.Delete_Templated folder=${VAR_DIR}/cars session=${new_leader_session} + +*** Keywords *** +Setup + [Documentation] Initialize resources, memorize car shard leader and followers. + SetupUtils.Setup_Utils_For_Setup_And_Teardown + CarPeople.Set_Variables_For_Shard shard_name=car diff --git a/csit/testplans/controller-clustering.txt b/csit/testplans/controller-clustering.txt index 04bf940874..2e8d23ba4a 100644 --- a/csit/testplans/controller-clustering.txt +++ b/csit/testplans/controller-clustering.txt @@ -8,3 +8,5 @@ integration/test/csit/suites/controller/Clustering_Datastore/buycar_failover.rob integration/test/csit/suites/controller/Clustering_Datastore/entity_isolate.robot integration/test/csit/suites/controller/singleton_service/global_rpc_kill.robot integration/test/csit/suites/controller/singleton_service/global_rpc_isolate.robot +integration/test/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot +integration/test/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot -- 2.36.6