csit/suites/controller/Clustering_Datastore/car_outage_corners.robot

   1 *** Settings ***
   2 Documentation     Cluster suite for testing minimal and sum-minimal member population behavior.
   3 ...
   4 ...               Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   5 ...
   6 ...               This program and the accompanying materials are made available under the
   7 ...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
   8 ...               and is available at http://www.eclipse.org/legal/epl-v10.html
   9 ...
  10 ...
  11 ...               This test stops majority of the followers and verifies car addition is not possible,
  12 ...               then resumes single follower (first from original list) and checks that addition works.
  13 ...               Then remaining members are brought up.
  14 ...               Leader member is always up and assumed to remain Leading during the whole suite run.
  15 ...
  16 ...               TODO: Use initial data to check more operations.
  17 ...               TODO: Perhaps merge with car_failover_crud suite.
  18 ...
  19 ...               Other modules and Shards (people, car-people) are not accessed by this suite.
  20 ...
  21 ...               All data is deleted at the end of the suite.
  22 ...               This suite expects car module to have a separate Shard.
  23 Suite Setup       Setup
  24 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  25 Default Tags      clustering    carpeople    critical
  26 Library           Collections
  27 Resource          ${CURDIR}/../../../libraries/CarPeople.robot
  28 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  29 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  30 Resource          ${CURDIR}/../../../libraries/TemplatedRequests.robot
  31 Variables         ${CURDIR}/../../../variables/Variables.py
  32
  33 *** Variables ***
  34 ${CAR_ITEMS}      50
  35 ${MINORITY_START_I}    300
  36 ${MAJORITY_START_I}    200
  37 ${MEMBER_START_TIMEOUT}    300s
  38 @{SHARD_NAME_LIST}    car
  39 ${VAR_DIR}        ${CURDIR}/../../../variables/carpeople/crud
  40 ${CLUSTER_DIR}    ${CURDIR}/../../../variables/clustering
  41
  42 *** Test Cases ***
  43 Stop_Majority_Of_The_Followers
  44     [Documentation]    Stop half plus one car Follower members and set reviving followers down (otherwsise tipping followers cannot join cluster).
  45     ...    Mark most of stopped members as explicitly down, to allow the surviving leader make progress.
  46     ClusterManagement.Stop_Members_From_List_Or_All    member_index_list=${list_of_stopping}    confirm=True
  47     FOR    ${index}    IN    @{list_of_reviving}
  48         ${data}    OperatingSystem.Get File    ${CLUSTER_DIR}/member_down.json
  49         ${member_ip} =    Collections.Get_From_Dictionary    ${ClusterManagement__index_to_ip_mapping}    ${index}
  50         ${data}    String.Replace String    ${data}    {member_ip}    ${member_ip}
  51         TemplatedRequests.Post_To_Uri    uri=jolokia    data=${data}    content_type=${HEADERS}    accept=${ACCEPT_EMPTY}    session=${car_leader_session}
  52     END
  53
  54 Attempt_To_Add_Cars_To_Leader
  55     [Documentation]    Adding cars should fail, as majority of Followers are down.
  56     ${status}    ${message} =    BuiltIn.Run_Keyword_And_Ignore_Error    TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}    iterations=${CAR_ITEMS}
  57     ...    iter_start=${MINORITY_START_I}
  58     # TODO: Is there a specific status and mesage to require in this scenario?
  59     # Previously it was 500, now the restconf call simply times out.
  60     BuiltIn.Log    ${message}
  61     BuiltIn.Should_Be_Equal    ${status}    FAIL
  62
  63 Start_Tipping_Follower
  64     [Documentation]    Start one Follower member without persisted data.
  65     ClusterManagement.Start_Members_From_List_Or_All    member_index_list=${list_of_tipping}    wait_for_sync=True    timeout=${MEMBER_START_TIMEOUT}
  66     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config    member_index_list=${list_of_majority}
  67
  68 Add_Cars_On_Tipping_Follower
  69     [Documentation]    Add cars on the tipping Follower.
  70     TemplatedRequests.Put_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_first_follower_session}    iterations=${CAR_ITEMS}    iter_start=${MAJORITY_START_I}
  71
  72 See_Cars_On_Existing_Members
  73     [Documentation]    On each up member: GET cars, should match the ones added on tipping Follower.
  74     FOR    ${session}    IN    @{list_of_majority}
  75         TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${MAJORITY_START_I}
  76     END
  77
  78 Start_Other_Followers
  79     [Documentation]    Start other followers without persisted data.
  80     ClusterManagement.Start_Members_From_List_Or_All    member_index_list=${list_of_reviving}    wait_for_sync=True    timeout=${MEMBER_START_TIMEOUT}
  81     BuiltIn.Wait_Until_Keyword_Succeeds    30s    2s    ClusterManagement.Verify_Leader_Exists_For_Each_Shard    shard_name_list=${SHARD_NAME_LIST}    shard_type=config
  82
  83 See_Cars_On_New_Follower_Leader
  84     [Documentation]    GET cars from a new follower to see that the current state was replicated.
  85     TemplatedRequests.Get_As_Json_Templated    folder=${VAR_DIR}/cars    session=${car_last_follower_session}    verify=True    iterations=${CAR_ITEMS}    iter_start=${MAJORITY_START_I}
  86
  87 Delete_Cars_On_Leader
  88     [Documentation]    Delete cars on Leader.
  89     TemplatedRequests.Delete_Templated    folder=${VAR_DIR}/cars    session=${car_leader_session}
  90
  91 *** Keywords ***
  92 Setup
  93     [Documentation]    Initialize resources, memorize shard leaders, pre-compute member lists.
  94     SetupUtils.Setup_Utils_For_Setup_And_Teardown    http_timeout=5
  95     CarPeople.Set_Variables_For_Shard    shard_name=car
  96     Set_Additional_Variables
  97
  98 Set_Additional_Variables
  99     [Documentation]    Compute various lists useful for test cases in this suite.
 100     # TODO: Migrate this Keyword to CarPeople Resource if more suites want that.
 101     ${last_follower_session} =    Collections.Get_From_List    ${car_follower_sessions}    -1
 102     BuiltIn.Set_Suite_Variable    \${car_last_follower_session}    ${last_follower_session}
 103     ${number_followers} =    BuiltIn.Get_Length    ${car_follower_indices}
 104     ${half_followers} =    BuiltIn.Evaluate    ${number_followers} / 2
 105     ${majority_follower_list} =    Collections.Get_Slice_From_List    ${car_follower_indices}    0    ${half_followers}
 106     ${leader_list} =    BuiltIn.Create_List    ${car_leader_index}
 107     ${majority_list} =    Collections.Combine_Lists    ${leader_list}    ${majority_follower_list}
 108     BuiltIn.Set_Suite_Variable    \${list_of_majority}    ${majority_list}
 109     ${tipping_list} =    Collections.Get_Slice_From_List    ${majority_follower_list}    0    1
 110     BuiltIn.Set_Suite_Variable    \${list_of_tipping}    ${tipping_list}
 111     ${revive_list} =    Collections.Get_Slice_From_List    ${car_follower_indices}    ${half_followers}    ${number_followers}
 112     BuiltIn.Set_Suite_Variable    \${list_of_reviving}    ${revive_list}
 113     ${stop_list} =    Collections.Combine_Lists    ${tipping_list}    ${revive_list}
 114     BuiltIn.Set_Suite_Variable    \${list_of_stopping}    ${stop_list}