test/csit/suites/clustering/datastore/140_recovery_restart_follower.robot

   1 *** Settings ***
   2 Documentation     This test kills any of the followers and verifies that when that follower is restarted it can join the cluster
   3 Default Tags  3-node-cluster
   4
   5 Library           Collections
   6 Library           ../../../libraries/RequestsLibrary.py
   7 Library           ../../../libraries/Common.py
   8 Library           ../../../libraries/CrudLibrary.py
   9 Library           ../../../libraries/SettingsLibrary.py
  10 Library           ../../../libraries/UtilLibrary.py
  11 Library           ../../../libraries/ClusterStateLibrary.py
  12 Resource          ../../../libraries/ClusterKeywords.txt
  13 Variables         ../../../variables/Variables.py
  14
  15 *** Variables ***
  16 ${REST_CONTEXT}    /restconf/config/
  17 ${CAR_SHARD}       shard-car-config
  18 ${NUM_CARS}     ${60}
  19
  20 *** Test Cases ***
  21 Stop All Controllers
  22     [Documentation]    Stop all the controllers in the cluster
  23     StopAllControllers    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${MEMBER1}    ${MEMBER2}    ${MEMBER3}
  24
  25 Clean All Journals
  26     [Documentation]    Clean the journals of all the controllers in the cluster
  27     CleanJournal    ${MEMBER1}    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}
  28     CleanJournal    ${MEMBER2}    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}
  29     CleanJournal    ${MEMBER3}    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}
  30
  31 Start All Controllers
  32     [Documentation]    Start all the controllers in the cluster
  33     ${rc}   StartAllControllers    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${RESTCONFPORT}
  34     ...    ${MEMBER1}    ${MEMBER2}    ${MEMBER3}
  35     Should Be True    ${rc}
  36
  37 Get car leader and followers
  38     ${CURRENT_CAR_LEADER}   Wait For Leader   ${CAR_SHARD}
  39     Set Suite Variable    ${CURRENT_CAR_LEADER}
  40     ${CAR_FOLLOWERS}  Get All Followers  ${CAR_SHARD}
  41     Set Suite Variable    ${CAR_FOLLOWERS}
  42
  43 Stop both of the followers
  44     StopAllControllers    ${USER_NAME}   ${PASSWORD}   ${KARAF_HOME}   @{CAR_FOLLOWERS}[0]    @{CAR_FOLLOWERS}[1]
  45     Wait Until Keyword Succeeds   30s  2s  Expect No Leader
  46
  47 Attempt to add a car from the leader
  48     [Documentation]    Should fail as both followers are down
  49     AddCar  ${CURRENT_CAR_LEADER}    ${PORT}    ${1}
  50     Sleep  2
  51     ${resp}    Getcars    ${CURRENT_CAR_LEADER}    ${PORT}    ${1}
  52     Should Not Be Equal As Strings    ${resp.status_code}    200
  53
  54 Restart the first follower
  55     StartController    @{CAR_FOLLOWERS}[0]   ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${PORT}
  56     Sleep    1
  57
  58 Add cars from the first follower
  59     Wait Until Keyword Succeeds   60s  2s  Add Cars And Verify   @{CAR_FOLLOWERS}[0]   ${NUM_CARS}  4s
  60
  61 Restart the second follower
  62     StartController    @{CAR_FOLLOWERS}[1]   ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${PORT}
  63
  64 Get all the cars from the second follower
  65     Wait Until Keyword Succeeds   60s  2s  Get Cars And Verify   @{CAR_FOLLOWERS}[1]   ${NUM_CARS}
  66
  67 *** Keywords ***
  68 Expect No Leader
  69     ${leader}   GetLeader   ${CAR_SHARD}   ${3}    ${1}    ${1}    ${PORT}     ${CURRENT_CAR_LEADER}
  70     Should Be Equal As Strings   ${leader}   None