csit/suites/controller/Clustering_Datastore/car_failover_crud.robot

   1 *** Settings ***
   2 Documentation     Suite mixing basic operations with restart of car Leader.
   3 ...
   4 ...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               This test kills the current leader of the "car" shard and then executes CRD
  12 ...               operations on the new leader and a new follower. The killed member is brought back.
  13 ...               This suite uses 3 different car sets, same size but different starting ID.
  14 ...
  15 ...               Other models and shards (people, car-people) are not accessed by this suite.
  16 ...
  17 ...               All data is deleted at the end of the suite.
  18 ...               This suite expects car module to have a separate Shard.
  19 Suite Setup       Setup
  20 Default Tags      clustering    carpeople    critical
  21 Library           Collections
  22 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  23 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  24 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  25 Variables         ${CURDIR}/../../../variables/Variables.py
  26
  27 *** Variables ***
  28 ${CAR_ITEMS}      30
  29 ${FOLLOWER_2NODE_START_I}    300
  30 ${LEADER_2NODE_START_I}    200
  31 ${MEMBER_START_TIMEOUT}    300s
  32 ${ORIGINAL_START_I}    100
  33 @{SHARD_NAME_LIST}    car
  34 ${VAR_DIR}        ${CURDIR}/../../../variables/carpeople/crud
  35
  36 *** Test Cases ***
  37 Add_Original_Cars_On_Old_Leader
  38     [Documentation]    Add initial cars on car Leader.
  39     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  40
  41 Kill_Original_Car_Leader
  42     [Documentation]    Kill the car Leader to cause a new leader to get elected.
  43     ${kill_list} =    BuiltIn.Create_List    ${car_leader_index}
  44     ClusterManagement.Kill_Members_From_List_Or_All    member_index_list=${kill_list}    confirm=True
  45
  46 Wait_For_New_Leader
  47     [Documentation]    Wait until new car Leader is elected.
  48     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    Set_Variables_For_Less_Nodes    member_index_list=${car_follower_indices}
  49
  50 See_Original_Cars_On_New_Leader
  51     [Documentation]    GET cars from new Leader, should be the initial ones.
  52     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  53
  54 See_Original_Cars_On_New_Followers
  55     [Documentation]    The same check on other existing member(s).
  56     : FOR    ${session}    IN    @{new_follower_sessions}
  57     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${ORIGINAL_START_I}
  58
  59 Delete_Original_Cars_On_New_Leader
  60     [Documentation]    Delete cars on the new Leader.
  61     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}
  62
  63 Add_Leader_Cars_On_New_Leader
  64     [Documentation]    Add cars on the new Leader.
  65     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    iterations=${CAR_ITEMS}    iter_start=${LEADER_2NODE_START_I}
  66
  67 See_Leader_Cars_On_New_Leader
  68     [Documentation]    GET cars from new Leader, should be the new ones.
  69     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${LEADER_2NODE_START_I}
  70
  71 See_Leader_Cars_On_New_Followers
  72     [Documentation]    The same check on other existing members.
  73     : FOR    ${session}    IN    @{new_follower_sessions}
  74     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${LEADER_2NODE_START_I}
  75
  76 Delete_Leader_Cars_On_New_First_Follower
  77     [Documentation]    Delete cars in new first Follower.
  78     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${new_first_follower_session}
  79
  80 Add_Follower_Cars_On_New_First_Follower
  81     [Documentation]    Add cars on the new first Follower.
  82     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_first_follower_session}    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
  83
  84 See_Folower_Cars_On_New_Leader
  85     [Documentation]    Get cars from the new Leader, should be the ones added on follower.
  86     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
  87
  88 See_Follower_Cars_On_New_Followers
  89     [Documentation]    The same check on other existing members.
  90     : FOR    ${session}    IN    @{new_follower_sessions}
  91     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
  92
  93 Start_Old_Car_Leader
  94     [Documentation]    Start the killed member without deleting the persisted data.
  95     ${revive_list} =    BuiltIn.Create_List    ${car_leader_index}
  96     ClusterManagement.Start_Members_From_List_Or_All    member_index_list=${revive_list}    wait_for_sync=True    timeout=${MEMBER_START_TIMEOUT}
  97     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config
  98
  99 See_Folower_Cars_On_Old_Leader
 100     [Documentation]    GET cars from the restarted member, should be the ones added on follower.
 101     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${FOLLOWER_2NODE_START_I}
 102
 103 Delete_Follower_Cars_On_New_Leader
 104     [Documentation]    Delete cars on the last Leader.
 105     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${new_leader_session}
 106
 107 *** Keywords ***
 108 Setup
 109     [Documentation]    Initialize resources, memorize car shard leader and followers.
 110     ClusterManagement.ClusterManagement_Setup
 111     CarPeople.Set_Variables_For_Shard    shard_name=car
 112
 113 Set_Variables_For_Less_Nodes
 114     [Arguments]    ${member_index_list}
 115     [Documentation]    Get current leader and followers for car shard, set additional suite variables.
 116     ${leader}    ${follower_list} =    ClusterManagement.Get_Leader_And_Followers_For_Shard    shard_name=car    shard_type=config    member_index_list=${member_index_list}
 117     ${leader_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${leader}
 118     BuiltIn.Set_Suite_Variable    \${new_leader_session}    ${leader_session}
 119     ${sessions} =    BuiltIn.Create_List
 120     : FOR    ${follower_index}    IN    @{follower_list}
 121     \    ${follower_session} =    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${follower_index}
 122     \    Collections.Append_To_List    ${sessions}    ${follower_session}
 123     BuiltIn.Set_Suite_Variable    \${new_follower_sessions}    ${sessions}
 124     ${first_follower_session} =    Collections.Get_From_List    ${sessions}    0
 125     BuiltIn.Set_Suite_Variable    \${new_first_follower_session}    ${first_follower_session}