Added test suites for cluster HA data recovery leader follower failover

[integration/test.git] / csit / libraries / WaitUtils.robot

diff --git a/csit/libraries/WaitUtils.robot b/csit/libraries/WaitUtils.robot
index e1f29b178a2368bf9605f5176a0cfd94dd3d3236..8d5252b7b05ec03ba093aecc595bf2ed635700c2 100644 (file)
--- a/csit/libraries/WaitUtils.robot
+++ b/csit/libraries/WaitUtils.robot
@@ -53,6 +53,14 @@ Create_Limiting_Stability_Safe_Stateful_Validator_From_Value_To_Overcome
     ${validator} =    ScalarClosures.Closure_From_Keyword_And_Arguments    WaitUtils.Limiting_Stability_Safe_Stateful_Validator_As_Keyword    state_holder    data_holder    valid_minimum=${valid_minimum}
     [Return]    ${validator}
 
+Excluding_Stability_Safe_Stateful_Validator_As_Keyword
+    [Arguments]    ${old_state}    ${data}    ${excluded_value}=-1
+    [Documentation]    Report failure if got the excluded value or if data value changed from last time. Useful to become validator.
+    ${new_state} =    BuiltIn.Set_Variable    ${data}
+    BuiltIn.Return_From_Keyword_If    ${data} == ${excluded_value}    ${new_state}    FAIL    Got the excluded value.
+    BuiltIn.Return_From_Keyword_If    ${data} != ${old_state}    ${new_state}    FAIL    Data value has changed.
+    [Return]    ${new_state}    PASS    Validated stable: ${data}
+
 WaitUtils__Check_Sanity_And_Compute_Derived_Times
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1
     [Documentation]    Common checks for argument values. Return times in seconds and deadline date implied by timeout time.
@@ -147,7 +155,10 @@ Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline
     \    # Getter may fail, but this Keyword should return state, so we need RKAIE.
     \    ${status}    ${data} =    BuiltIn.Run_Keyword_And_Ignore_Error    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_As_Is    ${getter}
     \    BuiltIn.Return_From_Keyword_If    '''${status}''' != '''PASS'''    ${state}    ${status}    Getter failed: ${data}
-    \    # TODO: Do we want to check time here?
+    \    # Is there enough time left?
+    \    ${status}    ${message} =    BuiltIn.Run_Keyword_And_Ignore_Error    WaitUtils__Is_Deadline_Reachable    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
+    \    ...    sleeps_left=${sleeps_left}    message=Last result: ${result}
+    \    BuiltIn.Return_From_Keyword_If    '''${status}''' != '''PASS'''    ${state}    ${status}    ${message}
     \    ${state}    ${status}    ${result} =    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_After_Replacing_First_Two_Arguments    ${safe_validator}
     \    ...    ${state}    ${data}
     \    # Validator may have reported failure.
@@ -187,7 +198,7 @@ Wait_For_Getter_And_Safe_Stateful_Validator_Consecutive_Success
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # The loop for failures.
-    : FOR    ${try}    IN RANGE    1    ${maximum_tries}
+    : FOR    ${try}    IN RANGE    1    ${maximum_tries}+1    # If maximum_tries is 3, for will go through 1, 2, and 3.
     \    ${state}    ${status}    ${result} =    Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
     \    ...    count=${count}    getter=${getter}    safe_validator=${safe_validator}    initial_state=${state}
     \    # Have we passed?
@@ -201,13 +212,16 @@ Wait_For_Getter_And_Safe_Stateful_Validator_Consecutive_Success
 Wait_For_Getter_Error_Or_Safe_Stateful_Validator_Consecutive_Success
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${getter}=${ScalarClosures__fail}    ${safe_validator}=${ScalarClosures__fail}    ${initial_state}=${NONE}
     [Documentation]    Analogue of Wait Until Keyword Succeeds, but it passes state of validator around and exits early on getter failure. Calls GASSVHTSCBD to verify data is "stable".
+    # If this ever fails, we want to know the exact inputs passed to it.
+    ${tmp}=    BuiltIn.Evaluate    int(${count})
+    BuiltIn.Log    count=${tmp}
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     # Maximum number of tries. TODO: Move to separate Keyword or add into CSACDT?
     ${maximum_tries} =    BuiltIn.Evaluate    math.ceil(${timeout_in_seconds} / ${period_in_seconds})    modules=math
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # The loop for failures.
-    : FOR    ${try}    IN RANGE    1    ${maximum_tries}
+    : FOR    ${try}    IN RANGE    1    ${maximum_tries}+1    # If maximum_tries is 3, for will go through 1, 2, and 3.
     \    ${state}    ${status}    ${result} =    Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
     \    ...    count=${count}    getter=${getter}    safe_validator=${safe_validator}    initial_state=${state}
     \    # Have we passed?