test/csit/suites/clustering/datastore/140_recovery_restart_follower.robot

   1 *** Settings ***
   2 Documentation     This test kills any of the followers and verifies that when that follower is restarted it can join the cluster
   3 Default Tags      3-node-cluster
   4 Library           Collections
   5 Library           ../../../libraries/RequestsLibrary.py
   6 Library           ../../../libraries/Common.py
   7 Library           ../../../libraries/CrudLibrary.py
   8 Library           ../../../libraries/SettingsLibrary.py
   9 Library           ../../../libraries/UtilLibrary.py
  10 Library           ../../../libraries/ClusterStateLibrary.py
  11 Resource          ../../../libraries/ClusterKeywords.txt
  12 Variables         ../../../variables/Variables.py
  13
  14 *** Variables ***
  15 ${REST_CONTEXT}    /restconf/config/
  16 ${CAR_SHARD}      shard-car-config
  17 ${NUM_CARS}       ${60}
  18
  19 *** Test Cases ***
  20 Stop All Controllers
  21     [Documentation]    Stop all the controllers in the cluster
  22     StopAllControllers    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${MEMBER1}    ${MEMBER2}    ${MEMBER3}
  23
  24 Clean All Journals
  25     [Documentation]    Clean the journals of all the controllers in the cluster
  26     CleanJournal    ${MEMBER1}    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}
  27     CleanJournal    ${MEMBER2}    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}
  28     CleanJournal    ${MEMBER3}    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}
  29
  30 Start All Controllers
  31     [Documentation]    Start all the controllers in the cluster
  32     ${rc}    StartAllControllers    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${RESTCONFPORT}    ${MEMBER1}
  33     ...    ${MEMBER2}    ${MEMBER3}
  34     Should Be True    ${rc}
  35
  36 Get car leader and followers
  37     ${CURRENT_CAR_LEADER}    Wait For Leader    ${CAR_SHARD}
  38     Set Suite Variable    ${CURRENT_CAR_LEADER}
  39     ${CAR_FOLLOWERS}    Get All Followers    ${CAR_SHARD}
  40     Set Suite Variable    ${CAR_FOLLOWERS}
  41
  42 Stop both of the followers
  43     StopAllControllers    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    @{CAR_FOLLOWERS}[0]    @{CAR_FOLLOWERS}[1]
  44     Wait Until Keyword Succeeds    30s    2s    Expect No Leader
  45
  46 Attempt to add a car from the leader
  47     [Documentation]    Should fail as both followers are down
  48     AddCar    ${CURRENT_CAR_LEADER}    ${PORT}    ${1}
  49     Sleep    2
  50     ${resp}    Getcars    ${CURRENT_CAR_LEADER}    ${PORT}    ${1}
  51     Should Not Be Equal As Strings    ${resp.status_code}    200
  52
  53 Restart the first follower
  54     StartController    @{CAR_FOLLOWERS}[0]    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${PORT}
  55     Sleep    1
  56
  57 Add cars from the first follower
  58     Wait Until Keyword Succeeds    60s    2s    Add Cars And Verify    @{CAR_FOLLOWERS}[0]    ${NUM_CARS}    4s
  59
  60 Restart the second follower
  61     StartController    @{CAR_FOLLOWERS}[1]    ${USER_NAME}    ${PASSWORD}    ${KARAF_HOME}    ${PORT}
  62
  63 Get all the cars from the second follower
  64     Wait Until Keyword Succeeds    60s    2s    Get Cars And Verify    @{CAR_FOLLOWERS}[1]    ${NUM_CARS}
  65
  66 *** Keywords ***
  67 Expect No Leader
  68     ${leader}    GetLeader    ${CAR_SHARD}    ${3}    ${1}    ${1}    ${PORT}
  69     ...    ${CURRENT_CAR_LEADER}
  70     Should Be Equal As Strings    ${leader}    None