csit/suites/openflowplugin/EntityOwhership/030_Cluster_Sync_Problems.robot

   1 *** Settings ***
   2 Documentation     Test suite for entity ownership service and openflowplugin. Makes changes on controller side (isolating cluster node)
   3 Suite Setup       Start Suite
   4 Suite Teardown    End Suite
   5 Test Template     Isolating Node Scenario
   6 Library           SSHLibrary
   7 Library           RequestsLibrary
   8 Library           XML
   9 Resource          ${CURDIR}/../../../libraries/Utils.robot
  10 Resource          ${CURDIR}/../../../libraries/OvsManager.robot
  11 Resource          ${CURDIR}/../../../libraries/ClusterManagement.robot
  12 Resource          ${CURDIR}/../../../libraries/ClusterOpenFlow.robot
  13 Resource          ${CURDIR}/../../../libraries/CompareStream.robot
  14 Library           Collections
  15
  16 *** Variables ***
  17 ${SWITCHES}       1
  18 # this is for mininet 2.2.1 ${START_CMD}    sudo mn --controller=remote,ip=${ODL_SYSTEM_1_IP} --controller=remote,ip=${ODL_SYSTEM_2_IP} --controller=remote,ip=${ODL_SYSTEM_3_IP} --topo linear,${SWITCHES} --switch ovsk,protocols=OpenFlow13
  19 ${START_CMD}      sudo mn --topo linear,${SWITCHES} --switch ovsk,protocols=OpenFlow13
  20 @{CONTROLLER_NODES}    ${ODL_SYSTEM_1_IP}    ${ODL_SYSTEM_2_IP}    ${ODL_SYSTEM_3_IP}
  21 @{cntls_idx_list}    ${1}    ${2}    ${3}
  22
  23 *** Test Cases ***
  24 Start Mininet To All Nodes
  25     [Template]    NONE
  26     ${mininet_conn_id}=    SSHLibrary.Open Connection    ${TOOLS_SYSTEM_IP}    prompt=${TOOLS_SYSTEM_PROMPT}
  27     BuiltIn.Set Suite Variable    ${mininet_conn_id}
  28     SSHLibrary.Login With Public Key    ${TOOLS_SYSTEM_USER}    ${USER_HOME}/.ssh/id_rsa    any
  29     SSHLibrary.Put File    ${CURDIR}/../../../libraries/DynamicMininet.py    .
  30     SSHLibrary.Execute Command    sudo ovs-vsctl set-manager ptcp:6644
  31     SSHLibrary.Execute Command    sudo mn -c
  32     SSHLibrary.Write    ${START_CMD}
  33     SSHLibrary.Read Until    mininet>
  34     ${cntls_list}    BuiltIn.Create List    ${ODL_SYSTEM_1_IP}    ${ODL_SYSTEM_2_IP}    ${ODL_SYSTEM_3_IP}
  35     ${switch_list}    BuiltIn.Create List
  36     : FOR    ${i}    IN RANGE    0    ${SWITCHES}
  37     \    ${sid}=    BuiltIn.Evaluate    ${i}+1
  38     \    Collections.Append To List    ${switch_list}    s${sid}
  39     BuiltIn.Set Suite Variable    ${active_member}    1
  40     OvsManager.Setup Clustered Controller For Switches    ${switch_list}    ${cntls_list}
  41     BuiltIn.Wait Until Keyword Succeeds    15s    1s    Are Switches Connected Topo
  42
  43 Switches To Be Connected To All Nodes
  44     [Documentation]    Initial check for correct connected topology.
  45     [Template]    NONE
  46     BuiltIn.Wait Until Keyword Succeeds    15x    1s    Check All Switches Connected To All Cluster Nodes
  47
  48 Isolating Owner Of Switch s1
  49     s1
  50     [Teardown]    Report_Failure_Due_To_Bug    6540
  51
  52 Switches Still Be Connected To All Nodes
  53     [Template]    NONE
  54     BuiltIn.Wait Until Keyword Succeeds    15x    1s    Check All Switches Connected To All Cluster Nodes
  55     [Teardown]    Report_Failure_Due_To_Bug    6540
  56
  57 Stop Mininet And Verify No Owners
  58     [Template]    NONE
  59     Utils.Stop Suite
  60     BuiltIn.Wait Until Keyword Succeeds    15x    1s    Check No Owners In Controller
  61     [Teardown]    Report_Failure_Due_To_Bug    6540
  62
  63 *** Keywords ***
  64 Start Suite
  65     ClusterManagement.ClusterManagement Setup
  66
  67 End Suite
  68     ClusterManagement.Flush Iptables From List Or All
  69     RequestsLibrary.Delete All Sessions
  70
  71 Are Switches Connected Topo
  72     [Documentation]    Checks wheather switches are connected to controller
  73     ${resp}=    ClusterManagement.Get From Member    ${OPERATIONAL_TOPO_API}/topology/flow:1    ${active_member}    access=${ACCEPT_XML}
  74     BuiltIn.Log    ${resp}
  75     ${count}=    XML.Get Element Count    ${resp}    xpath=node
  76     BuiltIn.Should Be Equal As Numbers    ${count}    ${SWITCHES}
  77
  78 Check All Switches Connected To All Cluster Nodes
  79     [Documentation]    Verifies all switches are connected to all cluster nodes
  80     OvsManager.Get Ovsdb Data
  81     : FOR    ${i}    IN RANGE    0    ${SWITCHES}
  82     \    ${sid}=    BuiltIn.Evaluate    ${i}+1
  83     \    OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_1_IP}    update_data=${False}
  84     \    OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_2_IP}    update_data=${False}
  85     \    OvsManager.Should Be Connected    s${sid}    ${ODL_SYSTEM_3_IP}    update_data=${False}
  86
  87 Isolating Node Scenario
  88     [Arguments]    ${switch_name}
  89     [Documentation]    Disconnect and connect master and follower and check switch data to be consistent
  90     ${idx}=    BuiltIn.Evaluate    str("${switch_name}"[1:])
  91     BuiltIn.Set Test Variable    ${idx}
  92     Isolate Switchs Old Owner    ${switch_name}
  93     Rejoin Switchs Old Owner    ${switch_name}
  94     Isolate Switchs Candidate    ${switch_name}
  95     Rejoin Switchs Candidate    ${switch_name}
  96     [Teardown]    Run Keyword If    "${isol_node}"!="${Empty}"    Rejoin Controller To The Cluster    ${isol_node}
  97
  98 Isolate Switchs Old Owner
  99     [Arguments]    ${switch_name}
 100     BuiltIn.Set Test Variable    ${isol_node}    ${Empty}
 101     Check Count Integrity    ${switch_name}    expected_controllers=3
 102     ${old_owner}    ${old_followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 103     ${old_master}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_owner}_IP}
 104     ${tmp_followers}=    BuiltIn.Create List    @{old_followers}
 105     Collections.Remove Values From List    ${tmp_followers}    ${old_owner}
 106     ${tmp_follower}=    Collections.Get From List    ${tmp_followers}    0
 107     BuiltIn.Set Suite Variable    ${active_member}    ${tmp_follower}
 108     Isolate Controller From The Cluster    ${old_owner}
 109     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${tmp_followers}
 110     BuiltIn.Set Test Variable    ${isol_node}    ${old_owner}
 111     ${new_master}=    BuiltIn.Wait Until Keyword Succeeds    10x    3s    Verify New Master Controller Node    ${switch_name}    ${old_master}
 112     ${owner}    ${followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 113     Collections.List Should Contain Value    ${old_followers}    ${owner}
 114     ${expected_controllers}=    CompareStream.Set_Variable_If_At_Least_Boron    3    2
 115     BuiltIn.Wait Until Keyword Succeeds    10x    3s    Check Count Integrity    ${switch_name}    expected_controllers=${expected_controllers}
 116     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
 117     BuiltIn.Set Suite Variable    ${active_member}    ${owner}
 118     BuiltIn.Set Test Variable    ${old_owner}
 119     BuiltIn.Set Test Variable    ${old_followers}
 120     BuiltIn.Set Test Variable    ${old_master}
 121     BuiltIn.Set Test Variable    ${owner}
 122     BuiltIn.Set Test Variable    ${new_master}
 123
 124 Rejoin Switchs Old Owner
 125     [Arguments]    ${switch_name}
 126     Rejoin Controller To The Cluster    ${old_owner}
 127     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event
 128     BuiltIn.Set Test Variable    ${isol_node}    ${Empty}
 129     BuiltIn.Wait Until Keyword Succeeds    20x    3s    Verify Follower Added    ${switch_name}    ${old_owner}
 130     ${new_owner}    ${new_followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 131     BuiltIn.Wait Until Keyword Succeeds    10x    3s    Check Count Integrity    ${switch_name}    expected_controllers=3
 132     BuiltIn.Should Be Equal    ${owner}    ${new_owner}
 133     Collections.List Should Contain Value    ${new_followers}    ${old_owner}
 134
 135 Isolate Switchs Candidate
 136     [Arguments]    ${switch_name}
 137     ${old_owner}    ${old_followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 138     ${old_follower}=    Collections.Get From List    ${old_followers}    0
 139     ${old_slave}=    BuiltIn.Set Variable    ${ODL_SYSTEM_${old_follower}_IP}
 140     ${tmp_members}=    BuiltIn.Create List    @{ClusterManagement__member_index_list}
 141     Collections.Remove Values From List    ${tmp_members}    ${old_follower}
 142     Isolate Controller From The Cluster    ${old_follower}
 143     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${tmp_members}
 144     BuiltIn.Set Test Variable    ${isol_cntl}    ${old_slave}
 145     ${expected_controllers}=    CompareStream.Set_Variable_If_At_Least_Boron    3    2
 146     BuiltIn.Wait Until Keyword Succeeds    10x    3s    Check Count Integrity    ${switch_name}    expected_controllers=${expected_controllers}
 147     ${owner}    ${followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 148     BuiltIn.Should Be Equal    ${owner}    ${old_owner}
 149     BuiltIn.Should Be Equal As Strings    ${new_master}    ${ODL_SYSTEM_${owner}_IP}
 150     BuiltIn.Set Test Variable    ${old_owner}
 151     BuiltIn.Set Test Variable    ${old_followers}
 152     BuiltIn.Set Test Variable    ${old_follower}
 153     BuiltIn.Set Test Variable    ${old_slave}
 154     BuiltIn.Set Test Variable    ${owner}
 155
 156 Rejoin Switchs Candidate
 157     [Arguments]    ${switch_name}
 158     Rejoin Controller To The Cluster    ${old_follower}
 159     ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event
 160     BuiltIn.Wait Until Keyword Succeeds    20x    3s    Verify Follower Added    ${switch_name}    ${old_follower}
 161     BuiltIn.Set Test Variable    ${isol_node}    ${Empty}
 162     BuiltIn.Wait Until Keyword Succeeds    10x    3s    Check Count Integrity    ${switch_name}    expected_controllers=3
 163     ${new_owner}    ${new_followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 164     BuiltIn.Should Be Equal    ${old_owner}    ${new_owner}
 165     Collections.List Should Contain Value    ${new_followers}    ${old_follower}
 166
 167 Rejoin Controller To The Cluster
 168     [Arguments]    ${isolated_node}
 169     ClusterManagement.Rejoin Member From List Or All    ${isolated_node}
 170     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}
 171
 172 Isolate Controller From The Cluster
 173     [Arguments]    ${isolated_node}
 174     ClusterManagement.Isolate Member From List Or All    ${isolated_node}
 175     [Teardown]    SSHLibrary.Switch Connection    ${mininet_conn_id}
 176
 177 Check Count Integrity
 178     [Arguments]    ${switch_name}    ${expected_controllers}=3
 179     [Documentation]    Every switch must have only one master and rest must be followers and together must be of expected nodes count
 180     ${idx}=    BuiltIn.Evaluate    "${switch_name}"[1:]
 181     ${owner}    ${candidates}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 182     ${count}=    BuiltIn.Get Length    ${candidates}
 183     BuiltIn.Should Be Equal As Numbers    ${expected_controllers}    ${count}
 184
 185 Check No Owners In Controller
 186     [Documentation]    Check there is no owners in controllers
 187     ${session} =    Resolve_Http_Session_For_Member    member_index=${active_member}
 188     ${data} =    TemplatedRequests.Get_As_Json_From_Uri    uri=${ENTITY_OWNER_URI}    session=${session}
 189     BuiltIn.Should Not Contain    ${data}    member
 190
 191 Verify New Master Controller Node
 192     [Arguments]    ${switch_name}    ${old_master}
 193     [Documentation]    Checks if given node is different from actual master
 194     ${idx}=    BuiltIn.Evaluate    "${switch_name}"[1:]
 195     ${owner}    ${followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 196     ${new_master}    BuiltIn.Set Variable    ${ODL_SYSTEM_${owner}_IP}
 197     BuiltIn.Should Not Be Equal    ${old_master}    ${new_master}
 198     Return From Keyword    ${new_master}
 199
 200 Verify Follower Added
 201     [Arguments]    ${switch_name}    ${expected_node}
 202     [Documentation]    Checks if given node is in the list of active followers
 203     ${idx}=    BuiltIn.Evaluate    "${switch_name}"[1:]
 204     ${owner}    ${followers}=    ClusterManagement.Get Owner And Candidates For Device    openflow:${idx}    openflow    ${active_member}
 205     Collections.List Should Contain Value    ${followers}    ${expected_node}