csit/suites/openflowplugin/EntityOwnership/020_Cluster_Node_Failure.robot

   1 *** Settings ***
   2 Documentation       Test suite for entity ownership service and openflowplugin. Makes changes on controller side (restart karaf)
   3
   4 Library             SSHLibrary
   5 Library             RequestsLibrary
   6 Library             XML
   7 Resource            ${CURDIR}/../../../libraries/SetupUtils.robot
   8 Resource            ${CURDIR}/../../../libraries/Utils.robot
   9 Resource            ${CURDIR}/../../../libraries/FlowLib.robot
  10 Resource            ${CURDIR}/../../../libraries/OvsManager.robot
  11 Resource            ${CURDIR}/../../../libraries/ClusterManagement.robot
  12 Resource            ${CURDIR}/../../../libraries/ClusterOpenFlow.robot
  13 Library             Collections
  14
  15 Suite Setup         Start Suite
  16 Suite Teardown      End Suite
  17 Test Setup          SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
  18 Test Template       Restarting Karaf Scenario
  19
  20
  21 *** Variables ***
  22 ${SWITCHES}         1
  23 ${START_CMD}        sudo mn --topo linear,${SWITCHES}
  24 ${KARAF_HOME}       ${WORKSPACE}${/}${BUNDLEFOLDER}
  25
  26
  27 *** Test Cases ***
  28 Switches To Be Connected To All Nodes
  29     [Documentation]    Initial check for correct connected topology.
  30     [Template]    NONE
  31     BuiltIn.Wait Until Keyword Succeeds    5x    3s    Check All Switches Connected To All Cluster Nodes
  32 Restarting Owner Of Switch s1
  33     s1
  34 Switches Still Be Connected To All Nodes
  35     [Template]    NONE
  36     BuiltIn.Wait Until Keyword Succeeds    5x    3s    Check All Switches Connected To All Cluster Nodes
  37
  38
  39 *** Keywords ***
  40 Start Suite
  41     SetupUtils.Setup_Utils_For_Setup_And_Teardown
  42     ${mininet_conn_id}=    SSHLibrary.Open Connection    ${TOOLS_SYSTEM_IP}    prompt=${TOOLS_SYSTEM_PROMPT}
  43     BuiltIn.Set Suite Variable    ${mininet_conn_id}
  44     SSHLibrary.Login With Public Key    ${TOOLS_SYSTEM_USER}    ${USER_HOME}/.ssh/id_rsa    any
  45     SSHLibrary.Put File    ${CURDIR}/../../../libraries/DynamicMininet.py    .
  46     SSHLibrary.Execute Command    sudo ovs-vsctl set-manager ptcp:6644
  47     SSHLibrary.Execute Command    sudo mn -c
  48     SSHLibrary.Write    ${START_CMD}
  49     SSHLibrary.Read Until    mininet>
  50     ${cntls_list}=    BuiltIn.Create List    ${ODL_SYSTEM_1_IP}    ${ODL_SYSTEM_2_IP}    ${ODL_SYSTEM_3_IP}
  51     ${switch_list}=    BuiltIn.Create List
  52     FOR    ${i}    IN RANGE    0    ${SWITCHES}
  53         ${sid}=    BuiltIn.Evaluate    ${i}+1
  54         Collections.Append To List    ${switch_list}    s${sid}
  55     END
  56     BuiltIn.Set Suite Variable    ${active_member}    1
  57     OvsManager.Setup Clustered Controller For Switches    ${switch_list}    ${cntls_list}
  58     BuiltIn.Wait Until Keyword Succeeds
  59     ...    10s
  60     ...    1s
  61     ...    ClusterOpenFlow.Verify Switch Connections Running On Member
  62     ...    ${SWITCHES}
  63     ...    1
  64
  65 End Suite
  66     RequestsLibrary.Delete All Sessions
  67     Utils.Stop Mininet
  68
  69 Check All Switches Connected To All Cluster Nodes
  70     [Documentation]    Verifies all switches are connected to all cluster nodes
  71     OvsManager.Get Ovsdb Data
  72     FOR    ${i}    IN RANGE    0    ${SWITCHES}
  73         ${sid}=    BuiltIn.Evaluate    ${i}+1
  74         OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_1_IP}    update_data=${False}
  75         OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_2_IP}    update_data=${False}
  76         OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_3_IP}    update_data=${False}
  77     END
  78
  79 Restarting Karaf Scenario
  80     [Documentation]    Disconnect and connect owner and successor and check switch data to be consistent
  81     [Arguments]    ${switch_name}
  82     ${idx}=    BuiltIn.Evaluate    str("${switch_name}"[1:])
  83     BuiltIn.Set Test Variable    ${idx}
  84     Kill Switchs Old Owner    ${switch_name}
  85     Restart Switchs Old Owner    ${switch_name}
  86     Kill Switchs Successor    ${switch_name}
  87     Restart Switchs Successor    ${switch_name}
  88     [Teardown]    Run Keyword If    "${stopped_karaf}"!="${Empty}"    Start Controller Node And Verify    ${stopped_karaf}
  89
  90 Kill Switchs Old Owner
  91     [Arguments]    ${switch_name}
  92     BuiltIn.Set Test Variable    ${stopped_karaf}    ${Empty}
  93     ${old_owner}    ${old_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device
  94     ...    openflow:${idx}
  95     ...    ${active_member}
  96     ${old_master}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_owner}_IP}
  97     ${active_member}=    Collections.Get From List    ${old_successors}    0
  98     BuiltIn.Set Suite Variable    ${active_member}
  99     Stop Controller Node And Verify    ${old_owner}
 100     BuiltIn.Set Test Variable    ${stopped_karaf}    ${old_owner}
 101     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${old_successors}
 102     ${new_master}=    BuiltIn.Wait Until Keyword Succeeds
 103     ...    5x
 104     ...    3s
 105     ...    Verify New Master Controller Node
 106     ...    ${switch_name}
 107     ...    ${old_master}
 108     ${owner}    ${successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device
 109     ...    openflow:${idx}
 110     ...    ${active_member}
 111     ...    ${old_successors}
 112     ...    after_stop=True
 113     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
 114     BuiltIn.Set Suite Variable    ${active_member}    ${owner}
 115     BuiltIn.Set Test Variable    ${old_owner}
 116     BuiltIn.Set Test Variable    ${old_successors}
 117     BuiltIn.Set Test Variable    ${old_master}
 118     BuiltIn.Set Test Variable    ${owner}
 119     BuiltIn.Set Test Variable    ${new_master}
 120
 121 Restart Switchs Old Owner
 122     [Arguments]    ${switch_name}
 123     Start Controller Node And Verify    ${old_owner}
 124     BuiltIn.Set Test Variable    ${stopped_karaf}    ${Empty}
 125     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event
 126     ${new_owner}    ${new_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device
 127     ...    openflow:${idx}
 128     ...    ${active_member}
 129     BuiltIn.Should Be Equal    ${owner}    ${new_owner}
 130
 131 Kill Switchs Successor
 132     [Arguments]    ${switch_name}
 133     ${old_owner}    ${old_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device
 134     ...    openflow:${idx}
 135     ...    ${active_member}
 136     ${old_successor}=    Collections.Get From List    ${old_successors}    0
 137     ${old_slave}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_successor}_IP}
 138     Stop Controller Node And Verify    ${old_successor}
 139     BuiltIn.Set Test Variable    ${stopped_karaf}    ${old_successor}
 140     ${tmp_candidates}=    BuiltIn.Create List    @{ClusterManagement__member_index_list}
 141     Collections.Remove Values From List    ${tmp_candidates}    ${old_successor}
 142     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${tmp_candidates}
 143     ${owner}    ${successor}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device
 144     ...    openflow:${idx}
 145     ...    ${active_member}
 146     ...    ${tmp_candidates}
 147     ...    after_stop=True
 148     BuiltIn.Should Be Equal    ${owner}    ${old_owner}
 149     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
 150     BuiltIn.Set Test Variable    ${old_owner}
 151     BuiltIn.Set Test Variable    ${old_successors}
 152     BuiltIn.Set Test Variable    ${old_successor}
 153     BuiltIn.Set Test Variable    ${old_slave}
 154     BuiltIn.Set Test Variable    ${owner}
 155
 156 Restart Switchs Successor
 157     [Arguments]    ${switch_name}
 158     Start Controller Node And Verify    ${old_successor}
 159     BuiltIn.Set Test Variable    ${stopped_karaf}    ${Empty}
 160     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event
 161     ${new_owner}    ${new_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device
 162     ...    openflow:${idx}
 163     ...    ${active_member}
 164     BuiltIn.Should Be Equal    ${old_owner}    ${new_owner}
 165
 166 Verify New Master Controller Node
 167     [Documentation]    Checks if given node is different from actual master
 168     [Arguments]    ${switch_name}    ${old_master}
 169     ${idx}=    BuiltIn.Evaluate    "${switch_name}"[1:]
 170     ${owner}    ${successors}=    ClusterManagement.Get Owner And Candidates For Device
 171     ...    openflow:${idx}
 172     ...    openflow
 173     ...    ${active_member}
 174     ${new_master}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${owner}_IP}
 175     BuiltIn.Should Not Be Equal    ${old_master}    ${new_master}
 176     RETURN    ${new_master}
 177
 178 Stop Controller Node And Verify
 179     [Documentation]    Stops the given node
 180     [Arguments]    ${node}
 181     ClusterManagement.Stop Single Member    ${node}
 182     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}
 183
 184 Start Controller Node And Verify
 185     [Documentation]    Starts the given node
 186     [Arguments]    ${node}
 187     ClusterManagement.Start Single Member    ${node}
 188     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}