Use consistent naming in vpn suite

[integration/test.git] / csit / libraries / WaitUtils.robot

diff --git a/csit/libraries/WaitUtils.robot b/csit/libraries/WaitUtils.robot
index e1f29b178a2368bf9605f5176a0cfd94dd3d3236..11338f382d0eee0f3fe0f1d59fe08d8487f9458f 100644 (file)
--- a/csit/libraries/WaitUtils.robot
+++ b/csit/libraries/WaitUtils.robot
@@ -8,10 +8,11 @@ Documentation     Robot keyword library (Resource) with several Keywords for mon
 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
 ...
 ...
 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
 ...
 ...

-...               BuiltIn.Wait_Until_Keyword_Succeeds is useful in avoiding unnecessary sleeps.
-...               But several usage cases need slightly different logic, here are Keywords for that.
+...               BuiltIn.Wait_Until_Keyword_Succeeds has two possible results: Fast pass or fail on timeout.
+...               Generally, keywords in this Resource also allow for some kind of fast failure condition.
+...               This usually requires more than a single keyword to run inside the iteration loop.
+...               This library uses ScalarClosures for plugging in specific (multiple) Keywords.

 ...
 ...

-...               This library uses ScalarClosures for plugging in specific Keywords.

 ...               Storing private state in suite variables is easy, but it can lead to hard-to-debug issues,
 ...               so this library tries to support explicit state passing.
 ...               Unfortunately, failing limits type of message to return,
 ...               Storing private state in suite variables is easy, but it can lead to hard-to-debug issues,
 ...               so this library tries to support explicit state passing.
 ...               Unfortunately, failing limits type of message to return,


@@ -26,7 +27,7 @@ Documentation     Robot keyword library (Resource) with several Keywords for mon
 ...               Safe Stateful Validator: Take ${state} and ${data} arguments. Return new state, validation status and comment/message.
 ...               TODO: Create a dummy closure for each type to be used as default value?
 ...
 ...               Safe Stateful Validator: Take ${state} and ${data} arguments. Return new state, validation status and comment/message.
 ...               TODO: Create a dummy closure for each type to be used as default value?
 ...

-...               TODO: Figure out a way to merge this with FaitForFailure.robot
+...               TODO: Figure out a way to merge this with WaitForFailure.robot

 ...               TODO: Add Keywords that are Safe (return state, success and message)
 ...               so that callers do not need to refresh state explicitly.
 Library           DateTime
 ...               TODO: Add Keywords that are Safe (return state, success and message)
 ...               so that callers do not need to refresh state explicitly.
 Library           DateTime


@@ -53,6 +54,14 @@ Create_Limiting_Stability_Safe_Stateful_Validator_From_Value_To_Overcome
     ${validator} =    ScalarClosures.Closure_From_Keyword_And_Arguments    WaitUtils.Limiting_Stability_Safe_Stateful_Validator_As_Keyword    state_holder    data_holder    valid_minimum=${valid_minimum}
     [Return]    ${validator}
 
     ${validator} =    ScalarClosures.Closure_From_Keyword_And_Arguments    WaitUtils.Limiting_Stability_Safe_Stateful_Validator_As_Keyword    state_holder    data_holder    valid_minimum=${valid_minimum}
     [Return]    ${validator}
 

+Excluding_Stability_Safe_Stateful_Validator_As_Keyword
+    [Arguments]    ${old_state}    ${data}    ${excluded_value}=-1
+    [Documentation]    Report failure if got the excluded value or if data value changed from last time. Useful to become validator.
+    ${new_state} =    BuiltIn.Set_Variable    ${data}
+    BuiltIn.Return_From_Keyword_If    ${data} == ${excluded_value}    ${new_state}    FAIL    Got the excluded value.
+    BuiltIn.Return_From_Keyword_If    ${data} != ${old_state}    ${new_state}    FAIL    Data value has changed.
+    [Return]    ${new_state}    PASS    Validated stable: ${data}
+

 WaitUtils__Check_Sanity_And_Compute_Derived_Times
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1
     [Documentation]    Common checks for argument values. Return times in seconds and deadline date implied by timeout time.
 WaitUtils__Check_Sanity_And_Compute_Derived_Times
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1
     [Documentation]    Common checks for argument values. Return times in seconds and deadline date implied by timeout time.


@@ -77,9 +86,25 @@ WaitUtils__Is_Deadline_Reachable
     ${time_minimal} =    BuiltIn.Evaluate    int(${sleeps_left}) * ${period_in_seconds}
     BuiltIn.Run_Keyword_If    ${time_minimal} >= ${time_deadline}    BuiltIn.Fail    Not possible to succeed within the deadline. ${message}
 
     ${time_minimal} =    BuiltIn.Evaluate    int(${sleeps_left}) * ${period_in_seconds}
     BuiltIn.Run_Keyword_If    ${time_minimal} >= ${time_deadline}    BuiltIn.Fail    Not possible to succeed within the deadline. ${message}
 

+Wait_For_Getter_Failure_Or_Stateless_Validator_Pass
+    [Arguments]    ${timeout}=60s    ${period}=1s    ${getter}=${ScalarClosures__fail}    ${stateless_validator}=${ScalarClosures__identity}
+    [Documentation]    Repeatedly run getter and plug its output to validator. If both pass, return validator message.
+    ...    If getter fails, fail. If validator fails, repeat in WUKS fashion (fail when timeout is exceeded).
+    ...    FIXME: Cover this keyword in WaitUtilTest.robot
+    ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}
+    ${iterations} =    BuiltIn.Evaluate    ${timeout_in_seconds} / ${period_in_seconds}
+    : FOR    ${i}    IN RANGE    ${iterations}
+    \    ${data} =    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_As_Is    ${getter}
+    \    ${status}    ${message} =    BuiltIn.Run_Keyword_And_Ignore_Error    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_After_Replacing_First_Argument    ${stateless_validator}
+    \    ...    ${data}
+    \    BuiltIn.Return_From_Keyword_If    "${status}" == "PASS"    ${message}
+    \    WaitUtils__Is_Deadline_Reachable    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}    message=Last validator message: ${message}
+    \    BuiltIn.Sleep    ${period_in_seconds} s
+    BuiltIn.Fail    Logic error, we should have returned before.
+

 Stateless_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline
     [Arguments]    ${date_deadline}=0    ${period_in_seconds}=1    ${count}=1    ${assertor}=${ScalarClosures__fail}
 Stateless_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline
     [Arguments]    ${date_deadline}=0    ${period_in_seconds}=1    ${count}=1    ${assertor}=${ScalarClosures__fail}

-    [Documentation]    Pass only if ${assertor} passes ${count} times in a row with ${period_in_seconds} between attempts; less standard arguments.
+    [Documentation]    Pass only if \${assertor} passes ${count} times in a row with ${period_in_seconds} between attempts; less standard arguments.

     ${result} =    BuiltIn.Set_Variable    No result yet.
     # Do we have enough time to succeed?
     ${sleeps} =    BuiltIn.Evaluate    ${count} - 1
     ${result} =    BuiltIn.Set_Variable    No result yet.
     # Do we have enough time to succeed?
     ${sleeps} =    BuiltIn.Evaluate    ${count} - 1


@@ -98,7 +123,7 @@ Stateless_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline
 
 Stateless_Assert_Closure_Has_To_Succeed_Consecutively
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${assertor}=${ScalarClosures__fail}
 
 Stateless_Assert_Closure_Has_To_Succeed_Consecutively
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${assertor}=${ScalarClosures__fail}

-    [Documentation]    Pass only if ${assertor} passes ${count} times in a row with ${period} between attempts; standard arguments.
+    [Documentation]    Pass only if \${assertor} passes ${count} times in a row with ${period} between attempts; standard arguments.

     # TODO: Put default values into variables for users to override at pybot invocation?
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     ${result} =    Stateless_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}    count=${count}    assertor=${assertor}
     # TODO: Put default values into variables for users to override at pybot invocation?
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     ${result} =    Stateless_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}    count=${count}    assertor=${assertor}


@@ -106,7 +131,7 @@ Stateless_Assert_Closure_Has_To_Succeed_Consecutively
 
 Stateful_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline
     [Arguments]    ${date_deadline}=0    ${period_in_seconds}=1    ${count}=1    ${assertor}=${ScalarClosures__fail}    ${initial_state}=${None}
 
 Stateful_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline
     [Arguments]    ${date_deadline}=0    ${period_in_seconds}=1    ${count}=1    ${assertor}=${ScalarClosures__fail}    ${initial_state}=${None}

-    [Documentation]    Pass only if ${assertor} passes ${count} times in a row with ${period} between attempts. Keep assertor state in local variable. Less standard arguments.
+    [Documentation]    Pass only if $\{assertor} passes ${count} times in a row with ${period} between attempts. Keep assertor state in local variable. Less standard arguments.

     # TODO: Put default values into variables for users to override.
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # TODO: Put default values into variables for users to override.
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}


@@ -126,7 +151,7 @@ Stateful_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline
 
 Stateful_Assert_Closure_Has_To_Succeed_Consecutively
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${assertor}=${ScalarClosures__fail}    ${initial_state}=${NONE}
 
 Stateful_Assert_Closure_Has_To_Succeed_Consecutively
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${assertor}=${ScalarClosures__fail}    ${initial_state}=${NONE}

-    [Documentation]    Pass only if ${assertor} passes ${count} times in a row with ${period} between attempts. Keep assertor state in local variable. Standard arguments.
+    [Documentation]    Pass only if \${assertor} passes ${count} times in a row with ${period} between attempts. Keep assertor state in local variable. Standard arguments.

     # TODO: Put default values into variables for users to override.
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     ${result} =    Stateful_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}    count=${count}    assertor=${assertor}    initial_state=${initial_state}
     # TODO: Put default values into variables for users to override.
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     ${result} =    Stateful_Assert_Closure_Has_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}    count=${count}    assertor=${assertor}    initial_state=${initial_state}


@@ -134,7 +159,7 @@ Stateful_Assert_Closure_Has_To_Succeed_Consecutively
 
 Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline
     [Arguments]    ${date_deadline}=0    ${period_in_seconds}=1    ${count}=1    ${getter}=${ScalarClosures__fail}    ${safe_validator}=${ScalarClosures__fail}    ${initial_state}=${NONE}
 
 Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline
     [Arguments]    ${date_deadline}=0    ${period_in_seconds}=1    ${count}=1    ${getter}=${ScalarClosures__fail}    ${safe_validator}=${ScalarClosures__fail}    ${initial_state}=${NONE}

-    [Documentation]    Pass only if consecutively ${count} times in a row with ${period} between attempts: ${getter} creates data and ${safe_validator} passes. Validator updates its state even if it reports failure. Always return validator state, status and message.
+    [Documentation]    Pass only if consecutively ${count} times in a row with ${period} between attempts: \${getter} creates data and \${safe_validator} passes. Validator updates its state even if it reports failure. Always return validator state, status and message.

     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # Do we have enough time to succeed?
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # Do we have enough time to succeed?


@@ -147,7 +172,10 @@ Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline
     \    # Getter may fail, but this Keyword should return state, so we need RKAIE.
     \    ${status}    ${data} =    BuiltIn.Run_Keyword_And_Ignore_Error    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_As_Is    ${getter}
     \    BuiltIn.Return_From_Keyword_If    '''${status}''' != '''PASS'''    ${state}    ${status}    Getter failed: ${data}
     \    # Getter may fail, but this Keyword should return state, so we need RKAIE.
     \    ${status}    ${data} =    BuiltIn.Run_Keyword_And_Ignore_Error    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_As_Is    ${getter}
     \    BuiltIn.Return_From_Keyword_If    '''${status}''' != '''PASS'''    ${state}    ${status}    Getter failed: ${data}

-    \    # TODO: Do we want to check time here?
+    \    # Is there enough time left?
+    \    ${status}    ${message} =    BuiltIn.Run_Keyword_And_Ignore_Error    WaitUtils__Is_Deadline_Reachable    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
+    \    ...    sleeps_left=${sleeps_left}    message=Last result: ${result}
+    \    BuiltIn.Return_From_Keyword_If    '''${status}''' != '''PASS'''    ${state}    ${status}    ${message}

     \    ${state}    ${status}    ${result} =    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_After_Replacing_First_Two_Arguments    ${safe_validator}
     \    ...    ${state}    ${data}
     \    # Validator may have reported failure.
     \    ${state}    ${status}    ${result} =    ScalarClosures.Run_Keyword_And_Collect_Garbage    ScalarClosures.Run_Closure_After_Replacing_First_Two_Arguments    ${safe_validator}
     \    ...    ${state}    ${data}
     \    # Validator may have reported failure.


@@ -182,12 +210,12 @@ Wait_For_Getter_And_Safe_Stateful_Validator_Consecutive_Success
     [Documentation]    Analogue of Wait Until Keyword Succeeds, but it passes state of validator around. Calls GASSVHTSCBD to verify data is "stable".
     # FIXME: Document that Safe Stateful Validator has to return state, status and message (and never fail)
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     [Documentation]    Analogue of Wait Until Keyword Succeeds, but it passes state of validator around. Calls GASSVHTSCBD to verify data is "stable".
     # FIXME: Document that Safe Stateful Validator has to return state, status and message (and never fail)
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}

-    # Maximum number of tries. TODO: Move to separate Keyword?
-    ${maximum_tries} =    BuiltIn.Evaluate    math.ceil(${timeout_in_seconds} / ${period_in_seconds})    modules=math
+    # Maximum number of sleeps. TODO: Move to separate Keyword?
+    ${maximum_sleeps} =    BuiltIn.Evaluate    math.ceil(${timeout_in_seconds} / ${period_in_seconds}) + 1    modules=math

     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # The loop for failures.
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # The loop for failures.

-    : FOR    ${try}    IN RANGE    1    ${maximum_tries}
+    : FOR    ${try}    IN RANGE    1    ${maximum_sleeps}+2    # If maximum_sleeps is 2, for will go through 1, 2, and 3.

     \    ${state}    ${status}    ${result} =    Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
     \    ...    count=${count}    getter=${getter}    safe_validator=${safe_validator}    initial_state=${state}
     \    # Have we passed?
     \    ${state}    ${status}    ${result} =    Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
     \    ...    count=${count}    getter=${getter}    safe_validator=${safe_validator}    initial_state=${state}
     \    # Have we passed?


@@ -201,13 +229,16 @@ Wait_For_Getter_And_Safe_Stateful_Validator_Consecutive_Success
 Wait_For_Getter_Error_Or_Safe_Stateful_Validator_Consecutive_Success
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${getter}=${ScalarClosures__fail}    ${safe_validator}=${ScalarClosures__fail}    ${initial_state}=${NONE}
     [Documentation]    Analogue of Wait Until Keyword Succeeds, but it passes state of validator around and exits early on getter failure. Calls GASSVHTSCBD to verify data is "stable".
 Wait_For_Getter_Error_Or_Safe_Stateful_Validator_Consecutive_Success
     [Arguments]    ${timeout}=60s    ${period}=1s    ${count}=1    ${getter}=${ScalarClosures__fail}    ${safe_validator}=${ScalarClosures__fail}    ${initial_state}=${NONE}
     [Documentation]    Analogue of Wait Until Keyword Succeeds, but it passes state of validator around and exits early on getter failure. Calls GASSVHTSCBD to verify data is "stable".

+    # If this ever fails, we want to know the exact inputs passed to it.
+    ${tmp}=    BuiltIn.Evaluate    int(${count})
+    BuiltIn.Log    count=${tmp}

     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}
     ${timeout_in_seconds}    ${period_in_seconds}    ${date_deadline} =    WaitUtils__Check_Sanity_And_Compute_Derived_Times    timeout=${timeout}    period=${period}    count=${count}

-    # Maximum number of tries. TODO: Move to separate Keyword or add into CSACDT?
-    ${maximum_tries} =    BuiltIn.Evaluate    math.ceil(${timeout_in_seconds} / ${period_in_seconds})    modules=math
+    # Maximum number of sleeps. TODO: Move to separate Keyword or add into CSACDT?
+    ${maximum_sleeps} =    BuiltIn.Evaluate    math.ceil(${timeout_in_seconds} / ${period_in_seconds})    modules=math

     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # The loop for failures.
     ${result} =    BuiltIn.Set_Variable    No result yet.
     ${state} =    BuiltIn.Set_Variable    ${initial_state}
     # The loop for failures.

-    : FOR    ${try}    IN RANGE    1    ${maximum_tries}
+    : FOR    ${try}    IN RANGE    1    ${maximum_sleeps}+2    # If maximum_sleeps is 2, for will go through 1, 2, and 3.

     \    ${state}    ${status}    ${result} =    Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
     \    ...    count=${count}    getter=${getter}    safe_validator=${safe_validator}    initial_state=${state}
     \    # Have we passed?
     \    ${state}    ${status}    ${result} =    Getter_And_Safe_Stateful_Validator_Have_To_Succeed_Consecutively_By_Deadline    date_deadline=${date_deadline}    period_in_seconds=${period_in_seconds}
     \    ...    count=${count}    getter=${getter}    safe_validator=${safe_validator}    initial_state=${state}
     \    # Have we passed?