Relax get entity timeouts
[integration/test.git] / csit / libraries / controller / DdbCommons.robot
index 5502a724185a1e571228dce07c87ef742883cf73..a121064a12dd8abd44cc343cf942de62ba63015a 100644 (file)
@@ -83,9 +83,9 @@ Get_Node_Indexes_For_The_ELM_Test
     ...    producer should be deployed.
     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    verify_restconf=False
     ${idx_from} =    BuiltIn.Set_Variable    ${leader}
-    ${idx_to} =    BuiltIn.Set_Variable    @{follower_list}[0]
-    ${idx_trans} =    BuiltIn.Set_Variable_If    "${leader_from}" == "remote" and "${leader_to}" == "remote"    @{follower_list}[1]    "${leader_from}" == "local"    ${leader}    "${leader_to}" == "local"
-    ...    @{follower_list}[0]
+    ${idx_to} =    BuiltIn.Set_Variable    ${follower_list}[0]
+    ${idx_trans} =    BuiltIn.Set_Variable_If    "${leader_from}" == "remote" and "${leader_to}" == "remote"    ${follower_list}[1]    "${leader_from}" == "local"    ${leader}    "${leader_to}" == "local"
+    ...    ${follower_list}[0]
     BuiltIn.Return_From_Keyword    ${idx_from}    ${idx_to}    ${idx_trans}
 
 Clean_Leader_Shutdown_Test_Templ
@@ -96,7 +96,8 @@ Clean_Leader_Shutdown_Test_Templ
     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
-    MdsalLowlevel.Shutdown_Shard_Replica    ${actual_leader}    ${shard_name}
+    BuiltIn.Comment    Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
+    ClusterAdmin.Remove_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${actual_leader}    ${shard_name}    ${shard_type}
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
     Check_Status_Of_First_Response    ${resp_list}
@@ -111,7 +112,8 @@ Clean_Leader_Shutdown_PrefBasedShard_Test_Templ
     ${producer_idx_as_list} =    BuiltIn.Create_List    ${producer_idx}
     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${producer_ip_as_list}    ${producer_idx_as_list}    ${ID_PREFIX}    ${TRANSACTION_PRODUCTION_TIME}    ${TRANSACTION_RATE_1K}
     BuiltIn.Sleep    ${SLEEP_AFTER_TRANSACTIONS_INIT}
-    MdsalLowlevel.Shutdown_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}
+    BuiltIn.Comment    Bug 8794 workaround: Use remove-shard-replica until shutdown starts behaving properly.
+    ClusterAdmin.Remove_Prefix_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${actual_leader}    ${shard_name}!!    ${shard_type}
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
     Check_Status_Of_First_Response    ${resp_list}
@@ -124,7 +126,7 @@ Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    verify_restconf=False
     ${follower_list_leangth} =    BuiltIn.Evaluate    ${NUM_ODL_SYSTEM}-1
     BuiltIn.Length_Should_Be    ${follower_list}    ${follower_list_leangth}
-    ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    @{follower_list}[0]
+    ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    ${follower_list}[0]
     BuiltIn.Return_From_Keyword    ${producer_idx}    ${leader}    ${follower_list}
 
 Leader_Isolation_Test_Templ
@@ -143,7 +145,7 @@ Leader_Isolation_Test_Templ
     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
     ${li_isolated}    BuiltIn.Set_Variable    ${True}
     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}    ${shard_type}    ${True}
-    ...    ${leader}    member_index_list=${follower_list}
+    ...    ${leader}    member_index_list=${follower_list}    verify_restconf=False
     ${heal_date} =    DateTime.Add_Time_To_Date    ${date_start}    ${heal_timeout}
     ${sleep_to_heal} =    Get_Seconds_To_Time    ${heal_date}
     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
@@ -176,7 +178,7 @@ Leader_Isolation_PrefBasedShard_Test_Templ
     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
     ${li_isolated}    BuiltIn.Set_Variable    ${True}
     BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}!!    ${shard_type}    ${True}
-    ...    ${leader}    member_index_list=${follower_list}
+    ...    ${leader}    member_index_list=${follower_list}    verify_restconf=False
     ${heal_date} =    DateTime.Add_Time_To_Date    ${date_start}    ${heal_timeout}
     ${sleep_to_heal} =    Get_Seconds_To_Time    ${heal_date}
     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
@@ -197,8 +199,9 @@ Leader_Isolation_Heal_Within_Rt
     [Documentation]    The leader isolation test case end if the heal happens within transaction timeout. All write transaction
     ...    producers shoudl finish without error.
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
-    : FOR    ${resp}    IN    @{resp_list}
-    \    TemplatedRequests.Check_Status_Code    @{resp}[2]
+    FOR    ${resp}    IN    @{resp_list}
+        TemplatedRequests.Check_Status_Code    ${resp}[2]
+    END
 
 Module_Leader_Isolation_Heal_Default
     [Arguments]    ${isolated_node}    ${time_to_finish}
@@ -214,8 +217,9 @@ Module_Leader_Isolation_Heal_Default
     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX2}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    chained_flag=${CHAINED_TX}
     ...    reset_globals=${False}
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
-    : FOR    ${resp}    IN    @{resp_list}
-    \    TemplatedRequests.Check_Status_Code    @{resp}[2]
+    FOR    ${resp}    IN    @{resp_list}
+        TemplatedRequests.Check_Status_Code    ${resp}[2]
+    END
 
 Prefix_Leader_Isolation_Heal_Default
     [Arguments]    ${isolated_node}    ${time_to_finish}
@@ -230,8 +234,9 @@ Prefix_Leader_Isolation_Heal_Default
     ${restart_producer_node_ip_as_list}    BuiltIn.Create_List    ${restart_producer_node_ip}
     MdsalLowlevelPy.Start_Produce_Transactions_On_Nodes    ${restart_producer_node_ip_as_list}    ${restart_producer_node_idx_as_list}    ${ID_PREFIX2}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    reset_globals=${False}
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
-    : FOR    ${resp}    IN    @{resp_list}
-    \    TemplatedRequests.Check_Status_Code    @{resp}[2]
+    FOR    ${resp}    IN    @{resp_list}
+        TemplatedRequests.Check_Status_Code    ${resp}[2]
+    END
 
 Client_Isolation_Test_Templ
     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
@@ -289,7 +294,7 @@ Ongoing_Transactions_Not_Failed_Yet
 Ongoing_Transactions_Failed
     [Documentation]    Verify if write-transaction failed.
     ${resp} =    MdsalLowlevelPy.Get_Next_Transactions_Response
-    Check_Status_Code    @{resp}[2]    explicit_status_codes=${TRANSACTION_FAILED}
+    Check_Status_Code    ${resp}[2]    explicit_status_codes=${TRANSACTION_FAILED}
 
 Get_Seconds_To_Time
     [Arguments]    ${date_in_future}
@@ -315,8 +320,9 @@ Listener_Stability_Test_Templ
     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
-    : FOR    ${resp}    IN    @{resp_list}
-    \    TemplatedRequests.Check_Status_Code    @{resp}[2]
+    FOR    ${resp}    IN    @{resp_list}
+        TemplatedRequests.Check_Status_Code    ${resp}[2]
+    END
     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Dtcl    ${idx_listen}
     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     BuiltIn.Should_Be_True    ${copy_matches}
@@ -339,8 +345,9 @@ Listener_Stability_PrefBasedShard_Test_Templ
     ...    ${shard_type}    ${True}    ${idx_from}    verify_restconf=False
     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
     ${resp_list} =    MdsalLowlevelPy.Wait_For_Transactions
-    : FOR    ${resp}    IN    @{resp_list}
-    \    TemplatedRequests.Check_Status_Code    @{resp}[2]
+    FOR    ${resp}    IN    @{resp_list}
+        TemplatedRequests.Check_Status_Code    ${resp}[2]
+    END
     ${copy_matches} =    MdsalLowlevel.Unsubscribe_Ddtl    ${idx_listen}
     ${subscribed} =    BuiltIn.Set_Variable    ${False}
     BuiltIn.Should_Be_True    ${copy_matches}
@@ -361,9 +368,10 @@ Remove_Prefix_Based_Shard_And_Verify
     ${all_indices} =    ClusterManagement.List_All_Indices
     ${node_to_trigger} =    Collections.Get_From_List    ${all_indices}    ${0}
     MdsalLowlevel.Remove_Prefix_Shard    ${node_to_trigger}    ${prefix}
-    : FOR    ${idx}    IN    @{all_indices}
-    \    BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${idx}    ${prefix}!!
-    \    ...    ${SHARD_TYPE}
+    FOR    ${idx}    IN    @{all_indices}
+        BuiltIn.Wait_Until_Keyword_Succeeds    60s    5s    Verify_Shard_Replica_Not_Present    ${idx}    ${prefix}!!
+        ...    ${SHARD_TYPE}
+    END
 
 Verify_Shard_Replica_Not_Present
     [Arguments]    ${member_index}    ${shard_name}    ${shard_type}
@@ -375,8 +383,8 @@ Verify_Shard_Replica_Not_Present
     BuiltIn.Should_Contain    ${text}    "status":404    javax.management.InstanceNotFoundException
 
 Restart_Test_Templ
-    [Documentation]    Kill every odl node and start again.
-    ClusterManagement.Kill_Members_From_List_Or_All
+    [Documentation]    Stop every odl node and start again.
+    ClusterManagement.Stop_Members_From_List_Or_All
     ClusterManagement.Clean_Directories_On_List_Or_All    tmp_dir=/tmp
     ClusterManagement.Start_Members_From_List_Or_All
     BuiltIn.Wait_Until_Keyword_Succeeds    300s    10s    ShardStability.Shards_Stability_Get_Details    ${DEFAULT_SHARD_LIST}    verify_restconf=True
@@ -385,6 +393,15 @@ Restart_Test_Templ
 Check_Status_Of_First_Response
     [Arguments]    ${resp_list}
     [Documentation]    Extract first item from the list, third item of the tuple and call TemplatedRequests to check the http status code.
-    # @{resp_list}[0][2] does not work
-    ${tuple} =    BuiltIn.Set_Variable    @{resp_list}[0]
-    TemplatedRequests.Check_Status_Code    @{tuple}[2]
+    # ${resp_list}[0][2] does not work
+    ${tuple} =    BuiltIn.Set_Variable    ${resp_list}[0]
+    TemplatedRequests.Check_Status_Code    ${tuple}[2]
+
+Change_Use_Tell_Based_Protocol
+    [Arguments]    ${status}    ${DATASTORE_CFG}
+    [Documentation]    Change status use-tell-based-protocol to True or False
+    ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^#use-tell-based-protocol=true/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
+    ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^#use-tell-based-protocol=false/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
+    BuiltIn.Run_Keyword_And_Return_If    "${status}" == "True"    ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^use-tell-based-protocol=false/use-tell-based-protocol=true/g" ${DATASTORE_CFG}
+    BuiltIn.Run_Keyword_And_Return_If    "${status}" == "False"    ClusterManagement.Check_Bash_Command_On_List_Or_All    sed -ie "s/^use-tell-based-protocol=true/use-tell-based-protocol=false/g" ${DATASTORE_CFG}
+    BuiltIn.Fail    Failure in status. Status can be True or False.