From 46e91d85423cebc13e19194acecf7c98c4890f53 Mon Sep 17 00:00:00 2001 From: Jamo Luhrsen Date: Mon, 26 Nov 2018 21:54:16 -0800 Subject: [PATCH] Add 10s polling to allow transactions to complete In some cases the check to ensure that all cars/people were populated in the config store happened just before the last few were there. This could be because of any number of reasons (slow external environment, testing in the middle of a GC operation maybe, etc) Adding a polling period will account for this and not give a false failure. Change-Id: Ic9c7456da37b5c2cca694b2affc77e048a2f7b31 JIRA:INTTEST-54 Signed-off-by: Jamo Luhrsen --- .../Clustering_Datastore/buycar_failover.robot | 18 ++++++++++++------ .../buycar_failover_isolation.robot | 18 ++++++++++++------ .../car_failover_crud.robot | 3 ++- .../car_failover_crud_isolation.robot | 3 ++- .../car_persistence_recovery.robot | 3 ++- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/csit/suites/controller/Clustering_Datastore/buycar_failover.robot b/csit/suites/controller/Clustering_Datastore/buycar_failover.robot index c3cb5d241d..047cec7f1c 100644 --- a/csit/suites/controller/Clustering_Datastore/buycar_failover.robot +++ b/csit/suites/controller/Clustering_Datastore/buycar_failover.robot @@ -34,14 +34,16 @@ Add_Cars_To_Leader_And_Verify ${car_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${car_items} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${car_items} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} + \ ... verify=True iterations=${car_items} Add_People_To_First_Follower_And_Verify [Documentation] Add all needed people to people first Follower, verify on each member. ${people_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 CarPeople.Add_Several_People session=${people_first_follower_session} iterations=${people_items} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} verify=True iterations=${people_items} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} + \ ... verify=True iterations=${people_items} Buy_Cars_On_Leader_And_Verify [Documentation] Buy some cars on the leader member. @@ -49,7 +51,8 @@ Buy_Cars_On_Leader_And_Verify CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 1 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Buy_Cars_On_Follower_And_Verify [Documentation] Buy some cars on the first follower member. @@ -57,7 +60,8 @@ Buy_Cars_On_Follower_And_Verify CarPeople.Buy_Several_Cars session=${car-people_first_follower_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 2 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Reboot_People_Leader [Documentation] Previous people Leader is rebooted. We should never stop the people first follower, this is where people are registered. @@ -71,7 +75,8 @@ Buy_Cars_On_Leader_After_Reboot_And_Verify CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 3 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Buy_Cars_On_Follower_After_Reboot_And_Verify [Documentation] Buy some cars on the first follower member. @@ -79,7 +84,8 @@ Buy_Cars_On_Follower_After_Reboot_And_Verify CarPeople.Buy_Several_Cars session=${car-people_first_follower_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 4 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Delete_All_CarPeople [Documentation] DELETE car-people container. No verification beyond http status. diff --git a/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot b/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot index ac2cf134dc..637ef9d21a 100644 --- a/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot +++ b/csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot @@ -34,14 +34,16 @@ Add_Cars_To_Leader_And_Verify ${car_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${car_items} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${car_items} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} + \ ... verify=True iterations=${car_items} Add_People_To_First_Follower_And_Verify [Documentation] Add all needed people to people first Follower, verify on each member. ${people_items} = BuiltIn.Evaluate ${CARPEOPLE_ITEMS} * 4 CarPeople.Add_Several_People session=${people_first_follower_session} iterations=${people_items} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} verify=True iterations=${people_items} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/people session=${session} + \ ... verify=True iterations=${people_items} Buy_Cars_On_Leader_And_Verify [Documentation] Buy some cars on the leader member. @@ -49,7 +51,8 @@ Buy_Cars_On_Leader_And_Verify CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 1 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Buy_Cars_On_Follower_And_Verify [Documentation] Buy some cars on the first follower member. @@ -57,7 +60,8 @@ Buy_Cars_On_Follower_And_Verify CarPeople.Buy_Several_Cars session=${car-people_first_follower_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 2 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Isolate_and_Rejoin_People_Leader [Documentation] Previous people Leader is isolated. We should never stop the people first follower, this is where people are registered. @@ -73,7 +77,8 @@ Buy_Cars_On_Leader_After_Rejoin_And_Verify CarPeople.Buy_Several_Cars session=${car-people_leader_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 3 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Buy_Cars_On_Follower_After_Rejoin_And_Verify [Documentation] Buy some cars on the first follower member. @@ -81,7 +86,8 @@ Buy_Cars_On_Follower_After_Rejoin_And_Verify CarPeople.Buy_Several_Cars session=${car-people_first_follower_session} iterations=${CARPEOPLE_ITEMS} iter_start=${iter_start} ${total_iterations} = BuiltIn.Evaluate 4 * ${CARPEOPLE_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} verify=True iterations=${total_iterations} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/car-people session=${session} + \ ... verify=True iterations=${total_iterations} Delete_All_CarPeople [Documentation] DELETE car-people container. No verification beyond http status. diff --git a/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot b/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot index b2014f8e1b..4deefc2efa 100644 --- a/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot +++ b/csit/suites/controller/Clustering_Datastore/car_failover_crud.robot @@ -40,7 +40,8 @@ Add_Original_Cars_On_Old_Leader_And_Verify [Documentation] Add initial cars on car Leader. TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} + \ ... verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} Stop_Original_Car_Leader [Documentation] Stop the car Leader to cause a new leader to get elected. diff --git a/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot b/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot index db80631c20..e3cbf8d842 100644 --- a/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot +++ b/csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot @@ -42,7 +42,8 @@ Add_Original_Cars_On_Old_Leader_And_Verify [Documentation] Add initial cars on car Leader. TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} + \ ... verify=True iterations=${CAR_ITEMS} iter_start=${ORIGINAL_START_I} Isolate_Original_Car_Leader [Documentation] Isolate the car Leader to cause a new leader to get elected. diff --git a/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot b/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot index 09cf87db6d..433e8afdf4 100644 --- a/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot +++ b/csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot @@ -32,7 +32,8 @@ Add_Cars_On_Leader_And_Verify [Documentation] Single big PUT to datastore to add cars to car Leader. TemplatedRequests.Put_As_Json_Templated folder=${VAR_DIR}/cars session=${car_leader_session} iterations=${CAR_ITEMS} : FOR ${session} IN @{ClusterManagement__session_list} - \ TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} verify=True iterations=${CAR_ITEMS} + \ BuiltIn.Wait_Until_Keyword_Succeeds 10s 2s TemplatedRequests.Get_As_Json_Templated folder=${VAR_DIR}/cars session=${session} + \ ... verify=True iterations=${CAR_ITEMS} Stop_All_Members [Documentation] Stop all controllers. -- 2.36.6