csit/suites/controller/Clustering_Datastore/buycar_failover.robot

   1 *** Settings ***
   2 Documentation     This test focuses on testing buy-car RPC over 3 Leader reboots.
   3 ...
   4 ...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               All purchases are against the same node, which is the first one to get rebooted.
  12 ...
  13 ...               All data is deleted at the end of the suite.
  14 ...               This suite expects car, people and car-people modules to have separate Shards.
  15 Suite Setup       Setup
  16 Default Tags      clustering    carpeople    critical
  17 Library           Collections
  18 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  19 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  20 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  21 Variables         ${CURDIR}/../../../variables/Variables.py
  22
  23 *** Variables ***
  24 ${CARPEOPLE_ITEMS}    ${100}
  25 ${MEMBER_START_TIMEOUT}    300s
  26 @{SHARD_NAME_LIST}    car    people    car-people
  27 ${VAR_DIR}        ${CURDIR}/../../../variables/carpeople/crud
  28
  29 *** Test Cases ***
  30 Add_Cars_To_Leader_And_Verify
  31     [Documentation]    Add all needed cars to car Leader, verify on each member.
  32     ${car_items} =    BuiltIn.Evaluate    ${CARPEOPLE_ITEMS} * 4
  33     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${car_items}
  34     : FOR    ${session}    IN    @{ClusterManagement__session_list}
  35     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${car_items}
  36
  37 Add_People_To_First_Follower_And_Verify
  38     [Documentation]    Add all needed people to people first Follower, verify on each member.
  39     ${people_items} =    BuiltIn.Evaluate    ${CARPEOPLE_ITEMS} * 4
  40     CarPeople.Add_Several_People    session=${people_first_follower_session}    iterations=${people_items}
  41     : FOR    ${session}    IN    @{ClusterManagement__session_list}
  42     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/people    session=${session}    verify=True    iterations=${people_items}
  43
  44 Buy_Cars_After_0_Reboots_And_Verify
  45     [Documentation]    Buy some cars on the test member.
  46     ${iter_start} =    BuiltIn.Evaluate    0 * ${CARPEOPLE_ITEMS} + 1
  47     CarPeople.Buy_Several_Cars    session=${buying_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
  48     ${total_iterations} =    BuiltIn.Evaluate    1 * ${CARPEOPLE_ITEMS}
  49     : FOR    ${session}    IN    @{ClusterManagement__session_list}
  50     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
  51
  52 Reboot_1
  53     [Documentation]    Previous car-people Leader is rebooted (without persistence cleanup).
  54     ${index_to_reboot} =    Collections.Remove_From_List    ${list_to_reboot}    0
  55     ${index_list} =    BuiltIn.Create_List    ${index_to_reboot}
  56     ClusterManagement.Kill_Members_From_List_Or_All    member_index_list=${index_list}    confirm=True
  57     ClusterManagement.Start_Members_From_List_Or_All    member_index_list=${index_list}    wait_for_sync=True    timeout=${MEMBER_START_TIMEOUT}
  58     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config
  59
  60 Buy_Cars_After_1_Reboots_And_Verify
  61     [Documentation]    Buy some cars on the test member.
  62     ${iter_start} =    BuiltIn.Evaluate    1 * ${CARPEOPLE_ITEMS} + 1
  63     CarPeople.Buy_Several_Cars    session=${buying_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
  64     ${total_iterations} =    BuiltIn.Evaluate    2 * ${CARPEOPLE_ITEMS}
  65     : FOR    ${session}    IN    @{ClusterManagement__session_list}
  66     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
  67
  68 Reboot_2
  69     [Documentation]    Previous car-people Leader is rebooted (without persistence cleanup).
  70     ${index_to_reboot} =    Collections.Remove_From_List    ${list_to_reboot}    0
  71     ${index_list} =    BuiltIn.Create_List    ${index_to_reboot}
  72     ClusterManagement.Kill_Members_From_List_Or_All    member_index_list=${index_list}    confirm=True
  73     ClusterManagement.Start_Members_From_List_Or_All    member_index_list=${index_list}    wait_for_sync=True    timeout=${MEMBER_START_TIMEOUT}
  74     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config
  75
  76 Buy_Cars_After_2_Reboots_And_Verify
  77     [Documentation]    Buy some cars on the test member.
  78     ${iter_start} =    BuiltIn.Evaluate    2 * ${CARPEOPLE_ITEMS} + 1
  79     CarPeople.Buy_Several_Cars    session=${buying_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
  80     ${total_iterations} =    BuiltIn.Evaluate    3 * ${CARPEOPLE_ITEMS}
  81     : FOR    ${session}    IN    @{ClusterManagement__session_list}
  82     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
  83
  84 Reboot_3
  85     [Documentation]    Previous car-people Leader is rebooted (without persistence cleanup).
  86     ${index_to_reboot} =    Collections.Remove_From_List    ${list_to_reboot}    0
  87     ${index_list} =    BuiltIn.Create_List    ${index_to_reboot}
  88     ClusterManagement.Kill_Members_From_List_Or_All    member_index_list=${index_list}    confirm=True
  89     ClusterManagement.Start_Members_From_List_Or_All    member_index_list=${index_list}    wait_for_sync=True    timeout=${MEMBER_START_TIMEOUT}
  90     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config
  91
  92 Buy_Cars_After_3_Reboots_And_Verify
  93     [Documentation]    Buy some cars on the test member.
  94     ${iter_start} =    BuiltIn.Evaluate    3 * ${CARPEOPLE_ITEMS} + 1
  95     CarPeople.Buy_Several_Cars    session=${buying_session}    iterations=${CARPEOPLE_ITEMS}    iter_start=${iter_start}
  96     ${total_iterations} =    BuiltIn.Evaluate    4 * ${CARPEOPLE_ITEMS}
  97     : FOR    ${session}    IN    @{ClusterManagement__session_list}
  98     \    TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/car-people    session=${session}    verify=True    iterations=${total_iterations}
  99
 100 Delete_All_CarPeople
 101     [Documentation]    DELETE car-people container. No verification beyond http status.
 102     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/car-people    session=${buying_session}
 103
 104 Delete_All_People
 105     [Documentation]    DELETE people container. No verification beyond http status.
 106     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/people    session=${people_leader_session}
 107
 108 Delete_All_Cars
 109     [Documentation]    DELETE cars container. No verification beyond http status.
 110     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}
 111
 112 *** Keywords ***
 113 Setup
 114     [Documentation]    Initialize resources, memorize shard leaders, compute item distribution.
 115     ClusterManagement.ClusterManagement_Setup
 116     Set_Variables_For_Shard    shard_name=car
 117     Set_Variables_For_Shard    shard_name=people
 118     Set_Variables_For_Shard    shard_name=car-people
 119     ${leader_list} =    BuiltIn.Create_List    ${car-people_leader_index}
 120     ${reboot_list} =    Collections.Combine_Lists    ${leader_list}    ${car-people_follower_indices}
 121     BuiltIn.Set_Suite_Variable    \${list_to_reboot}    ${reboot_list}
 122     BuiltIn.Set_Suite_Variable    \${buying_session}    ${car-people_leader_session}