Add 10s polling to allow transactions to complete 90/78190/4
authorJamo Luhrsen <jluhrsen@redhat.com>
Tue, 27 Nov 2018 05:54:16 +0000 (21:54 -0800)
committerSam Hague <shague@redhat.com>
Thu, 29 Nov 2018 02:36:24 +0000 (02:36 +0000)
In some cases the check to ensure that all cars/people
were populated in the config store happened just before
the last few were there. This could be because of
any number of reasons (slow external environment,
testing in the middle of a GC operation maybe, etc)
Adding a polling period will account for this and
not give a false failure.

Change-Id: Ic9c7456da37b5c2cca694b2affc77e048a2f7b31
JIRA:INTTEST-54
Signed-off-by: Jamo Luhrsen <jluhrsen@redhat.com>
csit/suites/controller/Clustering_Datastore/buycar_failover.robot
csit/suites/controller/Clustering_Datastore/buycar_failover_isolation.robot
csit/suites/controller/Clustering_Datastore/car_failover_crud.robot
csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot
csit/suites/controller/Clustering_Datastore/car_persistence_recovery.robot

index c3cb5d241d83064d4c99fe73d1e371ed9040141f..047cec7f1c6229fbf0a5024f0bcd1a1a50028b67 100644 (file)
@@ -34,14 +34,16 @@ Add_Cars_To_Leader_And_Verify
     ${car_items} =    BuiltIn.Evaluate    ${CARPEOPLE_ITEMS} * 4
     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${car_items}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${car_items}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}
+    \    ...    verify=True    iterations=${car_items}
 
 Add_People_To_First_Follower_And_Verify
     [Documentation]    Add all needed people to people first Follower, verify on each member.
     ${people_items} =    BuiltIn.Evaluate    ${CARPEOPLE_ITEMS} * 4
     CarPeople.Add_Several_People    session=${people_first_follower_session}    iterations=${people_items}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/people    session=${session}    verify=True    iterations=${people_items}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/people    session=${session}
+    \    ...    verify=True    iterations=${people_items}
 
 Buy_Cars_On_Leader_And_Verify
     [Documentation]    Buy some cars on the leader member.
@@ -49,7 +51,8 @@ Buy_Cars_On_Leader_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_leader_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    1 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Buy_Cars_On_Follower_And_Verify
     [Documentation]    Buy some cars on the first follower member.
@@ -57,7 +60,8 @@ Buy_Cars_On_Follower_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_first_follower_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    2 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Reboot_People_Leader
     [Documentation]    Previous people Leader is rebooted. We should never stop the people first follower, this is where people are registered.
@@ -71,7 +75,8 @@ Buy_Cars_On_Leader_After_Reboot_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_leader_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    3 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Buy_Cars_On_Follower_After_Reboot_And_Verify
     [Documentation]    Buy some cars on the first follower member.
@@ -79,7 +84,8 @@ Buy_Cars_On_Follower_After_Reboot_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_first_follower_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    4 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Delete_All_CarPeople
     [Documentation]    DELETE car-people container. No verification beyond http status.
index ac2cf134dc2acf6caa871c5e816d3a135ef72c44..637ef9d21ae6963b8b1d5ac225c160c5758f0866 100644 (file)
@@ -34,14 +34,16 @@ Add_Cars_To_Leader_And_Verify
     ${car_items} =    BuiltIn.Evaluate    ${CARPEOPLE_ITEMS} * 4
     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${car_items}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${car_items}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}
+    \    ...    verify=True    iterations=${car_items}
 
 Add_People_To_First_Follower_And_Verify
     [Documentation]    Add all needed people to people first Follower, verify on each member.
     ${people_items} =    BuiltIn.Evaluate    ${CARPEOPLE_ITEMS} * 4
     CarPeople.Add_Several_People    session=${people_first_follower_session}    iterations=${people_items}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/people    session=${session}    verify=True    iterations=${people_items}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/people    session=${session}
+    \    ...    verify=True    iterations=${people_items}
 
 Buy_Cars_On_Leader_And_Verify
     [Documentation]    Buy some cars on the leader member.
@@ -49,7 +51,8 @@ Buy_Cars_On_Leader_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_leader_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    1 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Buy_Cars_On_Follower_And_Verify
     [Documentation]    Buy some cars on the first follower member.
@@ -57,7 +60,8 @@ Buy_Cars_On_Follower_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_first_follower_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    2 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Isolate_and_Rejoin_People_Leader
     [Documentation]    Previous people Leader is isolated. We should never stop the people first follower, this is where people are registered.
@@ -73,7 +77,8 @@ Buy_Cars_On_Leader_After_Rejoin_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_leader_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    3 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Buy_Cars_On_Follower_After_Rejoin_And_Verify
     [Documentation]    Buy some cars on the first follower member.
@@ -81,7 +86,8 @@ Buy_Cars_On_Follower_After_Rejoin_And_Verify
     CarPeople.Buy_Several_Cars    session=${car-people_first_follower_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
     ${total_iterations} =    BuiltIn.Evaluate    4 * ${CARPEOPLE_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}
+    \    ...    verify=True    iterations=${total_iterations}
 
 Delete_All_CarPeople
     [Documentation]    DELETE car-people container. No verification beyond http status.
index b2014f8e1b736423c4aef4231d2c536f4c61d237..4deefc2efacca95e92b74e1f786db37d9b906199 100644 (file)
@@ -40,7 +40,8 @@ Add_Original_Cars_On_Old_Leader_And_Verify
     [Documentation]    Add initial cars on car Leader.
     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}
+    \    ...    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
 
 Stop_Original_Car_Leader
     [Documentation]    Stop the car Leader to cause a new leader to get elected.
index db80631c2004f368dac17f0768a619ad1aba811c..e3cbf8d842133833448aa71a07e7e849b9221d7d 100644 (file)
@@ -42,7 +42,8 @@ Add_Original_Cars_On_Old_Leader_And_Verify
     [Documentation]    Add initial cars on car Leader.
     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}
+    \    ...    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
 
 Isolate_Original_Car_Leader
     [Documentation]    Isolate the car Leader to cause a new leader to get elected.
index 09cf87db6d5761980b8bc6c13597e59fa6c6d55f..433e8afdf48f51eff2e6567066677023d3259dc3 100644 (file)
@@ -32,7 +32,8 @@ Add_Cars_On_Leader_And_Verify
     [Documentation]    Single big PUT to datastore to add cars to car Leader.
     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${CAR_ITEMS}
     : FOR    ${session}    IN    @{ClusterManagement__session_list}
-    \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}
+    \    BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}
+    \    ...    verify=True    iterations=${CAR_ITEMS}
 
 Stop_All_Members
     [Documentation]    Stop all controllers.