csit/suites/controller/Clustering_Datastore/car_failover_crud_isolation.robot

   1 *** Settings ***
   2 Documentation     Suite mixing basic operations with isolation of car Leader.
   3 ...
   4 ...               Copyright (c) 2017 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               This test isolates the current leader of the "car" shard and then executes CRD
  12 ...               operations on the new leader and a new follower. The isolated member is brought back.
  13 ...               This suite uses 3 different car sets, same size but different starting ID.
  14 ...
  15 ...               Other models and shards (people, car-people) are not accessed by this suite.
  16 ...
  17 ...               All data is deleted at the end of the suite.
  18 ...               This suite expects car module to have a separate Shard.
  19 Suite Setup       Setup
  20 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  21 Default Tags      clustering    carpeople    critical
  22 Library           Collections
  23 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  24 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  25 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  26 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  27 Variables         ${CURDIR}/../../../variables/Variables.py
  28
  29 *** Variables ***
  30 ${CAR_ITEMS}      30
  31 ${FOLLOWER_2NODE_START_I}    300
  32 ${LEADER_2NODE_START_I}    200
  33 ${MEMBER_START_TIMEOUT}    300s
  34 ${ORIGINAL_START_I}    100
  35 ${SHARD_TYPE}     config
  36 ${SHARD_NAME}     car
  37 @{SHARD_NAME_LIST}    ${SHARD_NAME}
  38 ${VAR_DIR}        ${CURDIR}/../../../variables/carpeople/crud
  39
  40 *** Test Cases ***
  41 Add_Original_Cars_On_Old_Leader_And_Verify
  42     [Documentation]    Add initial cars on car Leader.
  43     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  44     FOR    ${session}    IN    @{ClusterManagement__session_list}
  45         BuiltIn.Wait_Until_Keyword_Succeeds    10s    2s    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}
  46         ...    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  47     END
  48
  49 Isolate_Original_Car_Leader
  50     [Documentation]    Isolate the car Leader to cause a new leader to get elected.
  51     ClusterManagement.Isolate_Member_From_List_Or_All    ${car_leader_index}
  52
  53 Wait_For_New_Leader
  54     [Documentation]    Wait until new car Leader is elected.
  55     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Shard_Leader_Elected    ${SHARD_NAME}    ${SHARD_TYPE}    ${True}
  56     ...    ${car_leader_index}    member_index_list=${car_follower_indices}
  57     CarPeople.Set_Tmp_Variables_For_Shard_For_Nodes    member_index_list=${car_follower_indices}    shard_name=${SHARD_NAME}    shard_type=${SHARD_TYPE}
  58
  59 See_Original_Cars_On_New_Leader
  60     [Documentation]    GET cars from new Leader, should be the initial ones.
  61     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  62
  63 See_Original_Cars_On_New_Followers
  64     [Documentation]    The same check on other existing member(s).
  65     FOR    ${session}    IN    @{new_follower_sessions}
  66         TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  67     END
  68
  69 Delete_Original_Cars_On_New_Leader
  70     [Documentation]    Delete cars on the new Leader.
  71     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}
  72
  73 Add_Leader_Cars_On_New_Leader
  74     [Documentation]    Add cars on the new Leader.
  75     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    iterations=${CAR_ITEMS}    iter_start=${LEADER_2NODE_START_I}
  76
  77 See_Leader_Cars_On_New_Leader
  78     [Documentation]    GET cars from new Leader, should be the new ones.
  79     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${LEADER_2NODE_START_I}
  80
  81 See_Leader_Cars_On_New_Followers
  82     [Documentation]    The same check on other existing members.
  83     FOR    ${session}    IN    @{new_follower_sessions}
  84         TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${LEADER_2NODE_START_I}
  85     END
  86
  87 Delete_Leader_Cars_On_New_First_Follower
  88     [Documentation]    Delete cars in new first Follower.
  89     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${new_first_follower_session}
  90
  91 Add_Follower_Cars_On_New_First_Follower
  92     [Documentation]    Add cars on the new first Follower.
  93     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_first_follower_session}    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
  94
  95 See_Folower_Cars_On_New_Leader
  96     [Documentation]    Get cars from the new Leader, should be the ones added on follower.
  97     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
  98
  99 See_Follower_Cars_On_New_Followers
 100     [Documentation]    The same check on other existing members.
 101     FOR    ${session}    IN    @{new_follower_sessions}
 102         TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
 103     END
 104
 105 Rejoin_Old_Car_Leader
 106     [Documentation]    Rejoin the isolated member without deleting the persisted data.
 107     ClusterManagement.Rejoin_Member_From_List_Or_All    ${car_leader_index}
 108     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config
 109
 110 See_Folower_Cars_On_Old_Leader
 111     [Documentation]    GET cars from the restarted member, should be the ones added on follower.
 112     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
 113
 114 Delete_Follower_Cars_On_New_Leader
 115     [Documentation]    Delete cars on the last Leader.
 116     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}
 117
 118 *** Keywords ***
 119 Setup
 120     [Documentation]    Initialize resources, memorize car shard leader and followers.
 121     SetupUtils.Setup_Utils_For_Setup_And_Teardown
 122     CarPeople.Set_Variables_For_Shard    shard_name=car