csit/suites/openflowplugin/EntityOwnership/020_Cluster_Node_Failure.robot

   1 *** Settings ***
   2 Documentation     Test suite for entity ownership service and openflowplugin. Makes changes on controller side (restart karaf)
   3 Suite Setup       Start Suite
   4 Suite Teardown    End Suite
   5 Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
   6 Test Template     Restarting Karaf Scenario
   7 Library           SSHLibrary
   8 Library           RequestsLibrary
   9 Library           XML
  10 Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
  11 Resource          ${CURDIR}/../../../libraries/Utils.robot
  12 Resource          ${CURDIR}/../../../libraries/FlowLib.robot
  13 Resource          ${CURDIR}/../../../libraries/OvsManager.robot
  14 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  15 Resource          ${CURDIR}/../../../libraries/ClusterOpenFlow.robot
  16 Library           Collections
  17
  18 *** Variables ***
  19 ${SWITCHES}       1
  20 ${START_CMD}      sudo mn --topo linear,${SWITCHES}
  21 ${KARAF_HOME}     ${WORKSPACE}${/}${BUNDLEFOLDER}
  22
  23 *** Test Cases ***
  24 Switches To Be Connected To All Nodes
  25     [Documentation]    Initial check for correct connected topology.
  26     [Template]    NONE
  27     BuiltIn.Wait Until Keyword Succeeds    5x    3s    Check All Switches Connected To All Cluster Nodes
  28
  29 Restarting Owner Of Switch s1
  30     s1
  31
  32 Switches Still Be Connected To All Nodes
  33     [Template]    NONE
  34     BuiltIn.Wait Until Keyword Succeeds    5x    3s    Check All Switches Connected To All Cluster Nodes
  35
  36 *** Keywords ***
  37 Start Suite
  38     SetupUtils.Setup_Utils_For_Setup_And_Teardown
  39     ${mininet_conn_id}=    SSHLibrary.Open Connection    ${TOOLS_SYSTEM_IP}    prompt=${TOOLS_SYSTEM_PROMPT}
  40     BuiltIn.Set Suite Variable    ${mininet_conn_id}
  41     SSHLibrary.Login With Public Key    ${TOOLS_SYSTEM_USER}    ${USER_HOME}/.ssh/id_rsa    any
  42     SSHLibrary.Put File    ${CURDIR}/../../../libraries/DynamicMininet.py    .
  43     SSHLibrary.Execute Command    sudo ovs-vsctl set-manager ptcp:6644
  44     SSHLibrary.Execute Command    sudo mn -c
  45     SSHLibrary.Write    ${START_CMD}
  46     SSHLibrary.Read Until    mininet>
  47     ${cntls_list}    BuiltIn.Create List    ${ODL_SYSTEM_1_IP}    ${ODL_SYSTEM_2_IP}    ${ODL_SYSTEM_3_IP}
  48     ${switch_list}    BuiltIn.Create List
  49     FOR    ${i}    IN RANGE    0    ${SWITCHES}
  50         ${sid}=    BuiltIn.Evaluate    ${i}+1
  51         Collections.Append To List    ${switch_list}    s${sid}
  52     END
  53     BuiltIn.Set Suite Variable    ${active_member}    1
  54     OvsManager.Setup Clustered Controller For Switches    ${switch_list}    ${cntls_list}
  55     BuiltIn.Wait Until Keyword Succeeds    10s    1s    ClusterOpenFlow.Verify Switch Connections Running On Member    ${SWITCHES}    1
  56
  57 End Suite
  58     RequestsLibrary.Delete All Sessions
  59     Utils.Stop Mininet
  60
  61 Check All Switches Connected To All Cluster Nodes
  62     [Documentation]    Verifies all switches are connected to all cluster nodes
  63     OvsManager.Get Ovsdb Data
  64     FOR    ${i}    IN RANGE    0    ${SWITCHES}
  65         ${sid}=    BuiltIn.Evaluate    ${i}+1
  66         OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_1_IP}    update_data=${False}
  67         OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_2_IP}    update_data=${False}
  68         OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_3_IP}    update_data=${False}
  69     END
  70
  71 Restarting Karaf Scenario
  72     [Arguments]    ${switch_name}
  73     [Documentation]    Disconnect and connect owner and successor and check switch data to be consistent
  74     ${idx}=    BuiltIn.Evaluate    str("${switch_name}"[1:])
  75     BuiltIn.Set Test Variable    ${idx}
  76     Kill Switchs Old Owner    ${switch_name}
  77     Restart Switchs Old Owner    ${switch_name}
  78     Kill Switchs Successor    ${switch_name}
  79     Restart Switchs Successor    ${switch_name}
  80     [Teardown]    Run Keyword If    "${stopped_karaf}"!="${Empty}"    Start Controller Node And Verify    ${stopped_karaf}
  81
  82 Kill Switchs Old Owner
  83     [Arguments]    ${switch_name}
  84     BuiltIn.Set Test Variable    ${stopped_karaf}    ${Empty}
  85     ${old_owner}    ${old_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}
  86     ${old_master}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_owner}_IP}
  87     ${active_member}=    Collections.Get From List    ${old_successors}    0
  88     BuiltIn.Set Suite Variable    ${active_member}
  89     Stop Controller Node And Verify    ${old_owner}
  90     BuiltIn.Set Test Variable    ${stopped_karaf}    ${old_owner}
  91     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${old_successors}
  92     ${new_master}=    BuiltIn.Wait Until Keyword Succeeds    5x    3s    Verify New Master Controller Node    ${switch_name}    ${old_master}
  93     ${owner}    ${successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${old_successors}    after_stop=True
  94     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
  95     BuiltIn.Set Suite Variable    ${active_member}    ${owner}
  96     BuiltIn.Set Test Variable    ${old_owner}
  97     BuiltIn.Set Test Variable    ${old_successors}
  98     BuiltIn.Set Test Variable    ${old_master}
  99     BuiltIn.Set Test Variable    ${owner}
 100     BuiltIn.Set Test Variable    ${new_master}
 101
 102 Restart Switchs Old Owner
 103     [Arguments]    ${switch_name}
 104     Start Controller Node And Verify    ${old_owner}
 105     BuiltIn.Set Test Variable    ${stopped_karaf}    ${Empty}
 106     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event
 107     ${new_owner}    ${new_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}
 108     BuiltIn.Should Be Equal    ${owner}    ${new_owner}
 109
 110 Kill Switchs Successor
 111     [Arguments]    ${switch_name}
 112     ${old_owner}    ${old_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}
 113     ${old_successor}=    Collections.Get From List    ${old_successors}    0
 114     ${old_slave}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_successor}_IP}
 115     Stop Controller Node And Verify    ${old_successor}
 116     BuiltIn.Set Test Variable    ${stopped_karaf}    ${old_successor}
 117     ${tmp_candidates}=    BuiltIn.Create List    @{ClusterManagement__member_index_list}
 118     Collections.Remove Values From List    ${tmp_candidates}    ${old_successor}
 119     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${tmp_candidates}
 120     ${owner}    ${successor}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}    ${tmp_candidates}    after_stop=True
 121     BuiltIn.Should Be Equal    ${owner}    ${old_owner}
 122     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
 123     BuiltIn.Set Test Variable    ${old_owner}
 124     BuiltIn.Set Test Variable    ${old_successors}
 125     BuiltIn.Set Test Variable    ${old_successor}
 126     BuiltIn.Set Test Variable    ${old_slave}
 127     BuiltIn.Set Test Variable    ${owner}
 128
 129 Restart Switchs Successor
 130     [Arguments]    ${switch_name}
 131     Start Controller Node And Verify    ${old_successor}
 132     BuiltIn.Set Test Variable    ${stopped_karaf}    ${Empty}
 133     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event
 134     ${new_owner}    ${new_successors}=    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    openflow:${idx}    ${active_member}
 135     BuiltIn.Should Be Equal    ${old_owner}    ${new_owner}
 136
 137 Verify New Master Controller Node
 138     [Arguments]    ${switch_name}    ${old_master}
 139     [Documentation]    Checks if given node is different from actual master
 140     ${idx}=    BuiltIn.Evaluate    "${switch_name}"[1:]
 141     ${owner}    ${successors}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 142     ${new_master}    BuiltIn.Set Variable    ${ODL_SYSTEM_${owner}_IP}
 143     BuiltIn.Should Not Be Equal    ${old_master}    ${new_master}
 144     Return From Keyword    ${new_master}
 145
 146 Stop Controller Node And Verify
 147     [Arguments]    ${node}
 148     [Documentation]    Stops the given node
 149     ClusterManagement.Stop Single Member    ${node}
 150     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}
 151
 152 Start Controller Node And Verify
 153     [Arguments]    ${node}
 154     [Documentation]    Starts the given node
 155     ClusterManagement.Start Single Member    ${node}
 156     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}