csit/libraries/controller/DdbCommons.robot

   1 *** Settings ***
   2 Documentation     DOMDataBroker testing: Common keywords
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 Library           ${CURDIR}/../MdsalLowlevelPy.py
  10 Resource          ${CURDIR}/../ClusterAdmin.robot
  11 Resource          ${CURDIR}/../ClusterManagement.robot
  12 Resource          ${CURDIR}/../MdsalLowlevel.robot
  13 Resource          ${CURDIR}/../TemplatedRequests.robot
  14 Resource          ${CURDIR}/../WaitForFailure.robot
  15
  16 *** Variables ***
  17 ${SHARD_NAME}     default
  18 ${SHARD_TYPE}     config
  19 ${TRANSACTION_RATE_1K}    ${1000}
  20 ${DURATION_30S}    ${30}
  21 ${DURATION_10S}    ${10}
  22 ${ID_PREFIX}      prefix-
  23 ${TRANSACTION_TIMEOUT}    ${30}
  24 ${TRANSACTION_TIMEOUT_2X}    ${2*${TRANSACTION_TIMEOUT}}
  25 ${SIMPLE_TX}      ${False}
  26 ${CHAINED_TX}     ${True}
  27 ${HARD_TIMEOUT}    ${60}
  28 @{TRANSACTION_FAILED}    ${500}
  29
  30 *** Keywords ***
  31 Explicit_Leader_Movement_Test_Templ
  32     [Arguments]    ${leader_from}    ${leader_to}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  33     [Documentation]    Implements explicit leader movement test scenario.
  34     ${idx_from}    ${idx_to}    ${idx_trans} =    Get_Node_Indexes_For_The_ELM_Test    ${leader_from}    ${leader_to}
  35     ${ip_trans_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${idx_trans}_IP}
  36     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${ip_trans_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${False}
  37     ClusterAdmin.Make_Leader_Local    ${idx_to}    ${shard_name}    ${shard_type}
  38     ${new_leader}    ${new_followers} =    BuiltIn.Wait_Until_Keyword_Succeeds    10s    1s    ClusterManagement.Verify_Shard_Leader_Elected    ${shard_name}
  39     ...    ${shard_type}    ${True}    ${idx_from}
  40     BuiltIn.Should_Be_Equal    ${idx_to}    ${new_leader}
  41     ${resp_list} =    MdsalLowlevelPy.Wait_For_Write_Transactions
  42     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  43
  44 Get_Node_Indexes_For_The_ELM_Test
  45     [Arguments]    ${leader_from}    ${leader_to}
  46     [Documentation]    Return indexes for explicit leader movement test case, indexes of present to next leader node and index where transaction
  47     ...    producer should be deployed.
  48     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
  49     ${idx_from} =    BuiltIn.Set_Variable    ${leader}
  50     ${idx_to} =    BuiltIn.Set_Variable    @{follower_list}[0]
  51     ${idx_trans} =    BuiltIn.Set_Variable_If    "${leader_from}" == "remote" and "${leader_to}" == "remote"    @{follower_list}[1]    "${leader_from}" == "local"    ${leader}    "${leader_to}" == "local"
  52     ...    @{follower_list}[0]
  53     BuiltIn.Return_From_Keyword    ${idx_from}    ${idx_to}    ${idx_trans}
  54
  55 Clean_Leader_Shutdown_Test_Templ
  56     [Arguments]    ${leader_location}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
  57     [Documentation]    Implements clean leader shutdown test scenario.
  58     ${removed} =    BuiltIn.Set_Variable    ${False}
  59     ${producer_idx}    ${actual_leader} =    Get_Node_Indexes_For_Clean_Leader_Shutdown_Test    ${leader_location}
  60     ${producer_ip_as_list} =    BuiltIn.Create_List    ${ODL_SYSTEM_${producer_idx}_IP}
  61     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${producer_ip_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${False}
  62     ClusterAdmin.Remove_Shard_Replica    ${actual_leader}    ${shard_name}    member-${actual_leader}    ${shard_type}
  63     ${removed} =    BuiltIn.Set_Variable    ${True}
  64     ${resp_list} =    MdsalLowlevelPy.Wait_For_Write_Transactions
  65     TemplatedRequests.Check_Status_Code    @{resp_list}[0]
  66     [Teardown]    ClusterAdmin.Add_Shard_Replica    ${actual_leader}    ${shard_name}    ${shard_type}
  67
  68 Get_Node_Indexes_For_Clean_Leader_Shutdown_Test
  69     [Arguments]    ${leader_location}
  70     [Documentation]    Return indexes for clean leader shudown test case, index where transaction producer shoudl be deployed and a shard leader index.
  71     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
  72     ${follower_list_leangth} =    BuiltIn.Evaluate    ${NUM_ODL_SYSTEM}-1
  73     BuiltIn.Length_Should_Be    ${follower_list}    ${follower_list_leangth}
  74     ${producer_idx} =    BuiltIn.Set_Variable_If    "${leader_location}" == "local"    ${leader}    @{follower_list}[0]
  75     BuiltIn.Return_From_Keyword    ${producer_idx}    ${leader}
  76
  77 Leader_Isolation_Test_Templ
  78     [Arguments]    ${heal_timeout}
  79     [Documentation]    Implements leader isolation test scenario.
  80     ${producing_transactions_time} =    BuiltIn.Set_Variable    ${${heal_timeout}+60}
  81     ${all_indices} =    ClusterManagement.List_All_Indices
  82     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}    member_index_list=${all_indices}
  83     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
  84     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${ID_PREFIX}    ${producing_transactions_time}    ${TRANSACTION_RATE_1K}    chained_flag=${SIMPLE_TX}
  85     ${date_start} =    DateTime.Get_Current_Date
  86     ${date_end} =    DateTime.Add_Time_To_Date    ${date_start}    ${producing_transactions_time}
  87     ClusterManagement.Isolate_Member_From_List_Or_All    ${leader}
  88     BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
  89     ...    ${leader}    member_index_list=${follower_list}
  90     BuiltIn.Sleep    ${heal_timeout}
  91     ClusterManagement.Rejoin_Member_From_List_Or_All    ${leader}
  92     BuiltIn.Wait_Until_Keyword_Succeeds    20s    2s    ClusterManagement.Get_Leader_And_Followers_For_Shard    ${SHARD_NAME}    ${SHARD_TYPE}
  93     ${time_to_finish} =    Get_Seconds_To_Time    ${date_end}
  94     BuiltIn.Run_Keyword_If    ${heal_timeout} < ${TRANSACTION_TIMEOUT}    Leader_Isolation_Heal_Within_Tt
  95     ...    ELSE    Leader_Isolation_Heal_Default    ${time_to_finish}
  96
  97 Leader_Isolation_Heal_Within_Tt
  98     [Documentation]    The leader isolation test case end if the heal happens within transaction timeout. All write transaction
  99     ...    producers shoudl finish without error.
 100     ${resp_list} =    MdsalLowlevelPy.Wait_For_Write_Transactions
 101     : FOR    ${resp}    IN    @{resp_list}
 102     \    TemplatedRequests.Check_Status_Code    ${resp}
 103
 104 Leader_Isolation_Heal_Default
 105     [Arguments]    ${isolated_node}    ${time_to_finish}
 106     [Documentation]    The leader isolation test case end. The transaction producer on isolated node shoudl fail and should be restarted.
 107     Then all write transaction producers shoudl finish without error.
 108     ${resp} =    MdsalLowlevelPy.Get_Next_Write_Transactions_Response
 109     BuiltIn.Log    ${resp}
 110     ${restart_producer_node}    BuiltIn.Create_List    ${isolated_node}
 111     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${restart_producer_node}    ${ID_PREFIX}    ${time_to_finish}    ${TRANSACTION_RATE_1K}    chained_flag=${SIMPLE_TX}
 112     ${resp_list} =    MdsalLowlevelPy.Wait_For_Write_Transactions
 113     : FOR    ${resp}    IN    @{resp_list}
 114     \    TemplatedRequests.Check_Status_Code    ${resp}
 115
 116 Client_Isolation_Test_Templ
 117     [Arguments]    ${listener_node_role}    ${trans_chain_flag}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 118     [Documentation]    Implements client isolation test scenario.
 119     ${all_indices} =    ClusterManagement.List_All_Indices
 120     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 121     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 122     ${client_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 123     ${client_node_ip} =    ClusterManagement.Resolve_IP_Address_For_Member    ${client_node_dst}
 124     ${client_node_ip_as_list}    BuiltIn.Create_List    ${client_node_ip}
 125     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${client_node_ip_as_list}    ${ID_PREFIX}    ${DURATION_30S}    ${TRANSACTION_RATE_1K}    chained_flag=${trans_chain_flag}
 126     ${start_date}    DateTime.Get_Current_Date
 127     ${timeout_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${TRANSACTION_TIMEOUT}
 128     ${abort_date} =    DateTime.Add_Time_To_Date    ${start_date}    ${HARD_TIMEOUT}
 129     ClusterManagement.Isolate_Member_From_List_Or_All    ${client_node_dst}
 130     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${TRANSACTION_TIMEOUT}    1s    Write_Transactions_Not_Failed_Yet
 131     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    3s    1s    Write_Transactions_Failed
 132     ${abort_time}    Get_Seconds_To_Time    ${abort_date}
 133     WaitForFailure.Verify_Keyword_Does_Not_Fail_Within_Timeout    ${abort_time}    1s    Verify_Client_Aborted    ${False}
 134     WaitForFailure.Confirm_Keyword_Fails_Within_Timeout    3s    1s    Verify_Client_Aborted    ${True}
 135     [Teardown]    BuiltIn.Run Keywords    ClusterManagement.Rejoin_Member_From_List_Or_All    ${client_node_dst}
 136     ...    AND    MdsalLowlevelPy.Wait_For_Write_Transactions
 137
 138 Write_Transactions_Not_Failed_Yet
 139     [Documentation]    Verify that no write-transaction rpc finished, means they are still running.
 140     ${resp} =    MdsalLowlevelPy.Get_Next_Write_Transactions_Response
 141     BuiltIn.Should_Be_Equal    ${None}    ${resp}    ${resp} not expected.
 142
 143 Write_Transactions_Failed
 144     [Documentation]    Verify if write-transaction failed.
 145     ${resp} =    MdsalLowlevelPy.Get_Next_Write_Transactions_Response
 146     Check_Status_Code    ${resp}    explicit_status_codes=${TRANSACTION_FAILED}
 147
 148 Verify_Client_Aborted
 149     [Arguments]    ${exp_aborted}
 150     [Documentation]    To be reworked
 151     BuiltIn.Fail    TODO: Logic to be reworked
 152
 153 Get_Seconds_To_Time
 154     [Arguments]    ${date_in_future}
 155     [Documentation]    Return number of seconds remaining to ${date_in_future}.
 156     ${date_now} =    DateTime.Get_Current_Date
 157     ${duration} =    DateTime.Subtract_Date_From_Date    ${date_in_future}    ${date_now}
 158     BuiltIn.Run_Keyword_And_Return    BuiltIn.Convert_To_Integer    ${duration}
 159
 160 Remote_Listener_Test_Templ
 161     [Arguments]    ${listener_node_role}    ${shard_name}=${SHARD_NAME}    ${shard_type}=${SHARD_TYPE}
 162     [Documentation]    Implements remote listener test scenario.
 163     ${all_indices} =    ClusterManagement.List_All_Indices
 164     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=${shard_name}    shard_type=${shard_type}    member_index_list=${all_indices}
 165     ${follower1} =    Collections.Get_From_List    ${follower_list}    ${0}
 166     ${follower2} =    Collections.Get_From_List    ${follower_list}    ${1}
 167     ${listener_node_dst} =    BuiltIn.Set_Variable_If    "${listener_node_role}" == "leader"    ${leader}    ${follower1}
 168     MdsalLowlevel.Subscribe_Dtcl    ${listener_node_dst}
 169     ${all_ip_list} =    ClusterManagement.Resolve_IP_Address_For_Members    ${all_indices}
 170     MdsalLowlevelPy.Start_Write_Transactions_On_Nodes    ${all_ip_list}    ${ID_PREFIX}    ${DURATION_10S}    ${TRANSACTION_RATE_1K}    chained_flag=${SIMPLE_TX}
 171     ClusterAdmin.Make_Leader_Local    ${follower1}    ${shard_name}    ${shard_type}
 172     ${resp_list} =    MdsalLowlevelPy.Wait_For_Write_Transactions
 173     : FOR    ${resp}    IN    @{resp_list}
 174     \    TemplatedRequests.Check_Status_Code    ${resp}
 175     [Teardown]    MdsalLowlevel.Unsubscribe_Dtcl    ${listener_node_dst}