Add topology check in OpenFlow cluster test

Add 2 more switches connecting to first switch candidate instance.
Verify new inventory and topology during owner instance failure and recover.
verify topology when link between switches goes down and up.

Change-Id: I153a24599ce9478bc8e57cd161d46c606aa26bad
Signed-off-by: Luis Gomez <ecelgp@gmail.com>

diff --git a/csit/libraries/ClusterKeywords.robot b/csit/libraries/ClusterKeywords.robot
index 04ab7b613abd3c2d017e4c4aa2ccfa98c8f63a3d..dcf9bb11b86054ddbb187613b0c5c87047c51eed 100644 (file)
--- a/csit/libraries/ClusterKeywords.robot
+++ b/csit/libraries/ClusterKeywords.robot
@@ -25,7 +25,7 @@ Create Controller Sessions
     ${NUM_ODL_SYSTEM}=    Convert to Integer    ${NUM_ODL_SYSTEM}
     : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
     \    Log    Create Session ${ODL_SYSTEM_${i+1}_IP}
     ${NUM_ODL_SYSTEM}=    Convert to Integer    ${NUM_ODL_SYSTEM}
     : FOR    ${i}    IN RANGE    ${NUM_ODL_SYSTEM}
     \    Log    Create Session ${ODL_SYSTEM_${i+1}_IP}

-    \    Create Session    controller${i+1}    http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT}    auth=${AUTH}
+    \    RequestsLibrary.Create Session    controller${i+1}    http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT}    auth=${AUTH}

 
 Get Cluster Shard Status
     [Arguments]    ${controller_index_list}    ${shard_type}    ${shard}
 
 Get Cluster Shard Status
     [Arguments]    ${controller_index_list}    ${shard_type}    ${shard}


@@ -38,9 +38,9 @@ Get Cluster Shard Status
     ${leader}=    Set Variable    0
     ${follower_list}=    Create List
     : FOR    ${i}    IN    @{controller_index_list}
     ${leader}=    Set Variable    0
     ${follower_list}=    Create List
     : FOR    ${i}    IN    @{controller_index_list}

-    \    ${data}=    Get Data From URI    controller${i}    ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type}
+    \    ${data}=    Utils.Get Data From URI    controller${i}    ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type}

     \    Log    ${data}
     \    Log    ${data}

-    \    ${json}=    To Json    ${data}
+    \    ${json}=    RequestsLibrary.To Json    ${data}

     \    ${status}=    Get From Dictionary    &{json}[value]    RaftState
     \    Log    Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard}
     \    Run Keyword If    '${status}' == 'Leader'    Set Test Variable    ${leader}    ${i}
     \    ${status}=    Get From Dictionary    &{json}[value]    RaftState
     \    Log    Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard}
     \    Run Keyword If    '${status}' == 'Leader'    Set Test Variable    ${leader}    ${i}


@@ -54,9 +54,9 @@ Check Item Occurrence At URI In Cluster
     [Documentation]    Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list}
     ...    and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}.
     : FOR    ${i}    IN    @{controller_index_list}
     [Documentation]    Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list}
     ...    and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}.
     : FOR    ${i}    IN    @{controller_index_list}

-    \    ${data}    Get Data From URI    controller${i}    ${uri}
+    \    ${data}    Utils.Get Data From URI    controller${i}    ${uri}

     \    Log    ${data}
     \    Log    ${data}

-    \    Check Item Occurrence    ${data}    ${dictionary_item_occurrence}
+    \    Utils.Check Item Occurrence    ${data}    ${dictionary_item_occurrence}

 
 Put And Check At URI In Cluster
     [Arguments]    ${controller_index_list}    ${controller_index}    ${uri}    ${body}
 
 Put And Check At URI In Cluster
     [Arguments]    ${controller_index_list}    ${controller_index}    ${uri}    ${body}


@@ -89,15 +89,15 @@ Kill Multiple Controllers
     [Arguments]    @{controller_index_list}
     [Documentation]    Give this keyword a scalar or list of controllers to be stopped.
     : FOR    ${i}    IN    @{controller_index_list}
     [Arguments]    @{controller_index_list}
     [Documentation]    Give this keyword a scalar or list of controllers to be stopped.
     : FOR    ${i}    IN    @{controller_index_list}

-    \    ${output}=    Run Command On Controller    ${ODL_SYSTEM_${i}_IP}    ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
-    \    Controller Down Check    ${ODL_SYSTEM_${i}_IP}
+    \    ${output}=    Utils.Run Command On Controller    ${ODL_SYSTEM_${i}_IP}    ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
+    \    ClusterKeywords.Controller Down Check    ${ODL_SYSTEM_${i}_IP}

 
 Start Multiple Controllers
     [Arguments]    ${timeout}    @{controller_index_list}
     [Documentation]    Give this keyword a scalar or list of controllers to be started.
     : FOR    ${i}    IN    @{controller_index_list}
 
 Start Multiple Controllers
     [Arguments]    ${timeout}    @{controller_index_list}
     [Documentation]    Give this keyword a scalar or list of controllers to be started.
     : FOR    ${i}    IN    @{controller_index_list}

-    \    ${output}=    Run Command On Controller    ${ODL_SYSTEM_${i}_IP}    ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
-    \    Wait For Controller Sync    ${timeout}    ${ODL_SYSTEM_${i}_IP}
+    \    ${output}=    Utils.Run Command On Controller    ${ODL_SYSTEM_${i}_IP}    ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
+    \    ClusterKeywords.Wait For Controller Sync    ${timeout}    ${ODL_SYSTEM_${i}_IP}

 
 Get Controller List
     [Arguments]    ${exclude_controller}=${EMPTY}
 
 Get Controller List
     [Arguments]    ${exclude_controller}=${EMPTY}

diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot
index 4aaa55c3790a16fd7795dd0eb0b6cdb6cfe0f3b8..b96fc1c9b269f3b6b027acdbc577124030eb7ec6 100644 (file)
--- a/csit/libraries/ClusterOpenFlow.robot
+++ b/csit/libraries/ClusterOpenFlow.robot
@@ -15,10 +15,10 @@ ${operational_port_1}    ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector
 Check OpenFlow Shards Status
     [Arguments]    ${controller_index_list}
     [Documentation]    Check Status for all shards in OpenFlow application.
 Check OpenFlow Shards Status
     [Arguments]    ${controller_index_list}
     [Documentation]    Check Status for all shards in OpenFlow application.

-    ${inv_conf_leader}    ${inv_conf_followers_list}    Get Cluster Shard Status    ${controller_index_list}    config    inventory
-    ${inv_oper_leader}    ${inv_oper_followers_list}    Get Cluster Shard Status    ${controller_index_list}    operational    inventory
-    ${topo_oper_leader}    ${topo_oper_followers_list}    Get Cluster Shard Status    ${controller_index_list}    operational    topology
-    ${owner_oper_leader}    ${owner_oper_followers_list}    Get Cluster Shard Status    ${controller_index_list}    operational    entity-ownership
+    ${inv_conf_leader}    ${inv_conf_followers_list}    ClusterKeywords.Get Cluster Shard Status    ${controller_index_list}    config    inventory
+    ${inv_oper_leader}    ${inv_oper_followers_list}    ClusterKeywords.Get Cluster Shard Status    ${controller_index_list}    operational    inventory
+    ${topo_oper_leader}    ${topo_oper_followers_list}    ClusterKeywords.Get Cluster Shard Status    ${controller_index_list}    operational    topology
+    ${owner_oper_leader}    ${owner_oper_followers_list}    ClusterKeywords.Get Cluster Shard Status    ${controller_index_list}    operational    entity-ownership

     Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
     Log    operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list}
     Log    operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list}
     Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
     Log    operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list}
     Log    operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list}


@@ -27,7 +27,7 @@ Check OpenFlow Shards Status
 Check OpenFlow Shards Status After Cluster Event
     [Arguments]    ${controller_index_list}
     [Documentation]    Check Shards Status after some cluster event.
 Check OpenFlow Shards Status After Cluster Event
     [Arguments]    ${controller_index_list}
     [Documentation]    Check Shards Status after some cluster event.

-    Wait Until Keyword Succeeds    90s    1s    Check OpenFlow Shards Status    ${controller_index_list}
+    Wait Until Keyword Succeeds    90s    1s    ClusterOpenFlow.Check OpenFlow Shards Status    ${controller_index_list}

 
 Get Cluster Entity Owner For Openflow
     [Arguments]    ${controller_index_list}    ${device_type}    ${device}
 
 Get Cluster Entity Owner For Openflow
     [Arguments]    ${controller_index_list}    ${device_type}    ${device}


@@ -35,17 +35,17 @@ Get Cluster Entity Owner For Openflow
     ...    ${device_type} is normally openflow.
     ${length}=    Get Length    ${controller_index_list}
     ${candidates_list}=    Create List
     ...    ${device_type} is normally openflow.
     ${length}=    Get Length    ${controller_index_list}
     ${candidates_list}=    Create List

-    ${data}=    Get Data From URI    controller@{controller_index_list}[0]    /restconf/operational/entity-owners:entity-owners
+    ${data}=    Utils.Get Data From URI    controller@{controller_index_list}[0]    /restconf/operational/entity-owners:entity-owners

     Log    ${data}
     ${data}=    Replace String    ${data}    /general-entity:entity[general-entity:name='    ${EMPTY}
     ${clear_data}=    Replace String    ${data}    ']    ${EMPTY}
     Log    ${clear_data}
     Log    ${data}
     ${data}=    Replace String    ${data}    /general-entity:entity[general-entity:name='    ${EMPTY}
     ${clear_data}=    Replace String    ${data}    ']    ${EMPTY}
     Log    ${clear_data}

-    ${json}=    To Json    ${clear_data}
+    ${json}=    RequestsLibrary.To Json    ${clear_data}

     ${entity_type_list}=    Get From Dictionary    &{json}[entity-owners]    entity-type
     ${entity_type_index}=    Get Index From List Of Dictionaries    ${entity_type_list}    type    ${device_type}
     Should Not Be Equal    ${entity_type_index}    -1    No Entity Owner found for ${device_type}
     ${entity_list}=    Get From Dictionary    @{entity_type_list}[${entity_type_index}]    entity
     ${entity_type_list}=    Get From Dictionary    &{json}[entity-owners]    entity-type
     ${entity_type_index}=    Get Index From List Of Dictionaries    ${entity_type_list}    type    ${device_type}
     Should Not Be Equal    ${entity_type_index}    -1    No Entity Owner found for ${device_type}
     ${entity_list}=    Get From Dictionary    @{entity_type_list}[${entity_type_index}]    entity

-    ${entity_index}=    Get Index From List Of Dictionaries    ${entity_list}    id    ${device}
+    ${entity_index}=    Utils.Get Index From List Of Dictionaries    ${entity_list}    id    ${device}

     Should Not Be Equal    ${entity_index}    -1    Device ${device} not found in Entity Owner ${device_type}
     ${entity_owner}=    Get From Dictionary    @{entity_list}[${entity_index}]    owner
     Should Not Be Empty    ${entity_owner}    No owner found for ${device}
     Should Not Be Equal    ${entity_index}    -1    Device ${device} not found in Entity Owner ${device_type}
     ${entity_owner}=    Get From Dictionary    @{entity_list}[${entity_index}]    owner
     Should Not Be Empty    ${entity_owner}    No owner found for ${device}


@@ -63,28 +63,29 @@ Get Cluster Entity Owner For Openflow
     [Return]    ${owner}    ${candidates_list}
 
 Get OpenFlow Entity Owner Status For One Device
     [Return]    ${owner}    ${candidates_list}
 
 Get OpenFlow Entity Owner Status For One Device

-    [Arguments]    ${controller_index_list}
+    [Arguments]    ${controller_index_list}    ${device}

     [Documentation]    Check Entity Owner Status and identify owner and candidate.
     [Documentation]    Check Entity Owner Status and identify owner and candidate.

-    ${owner}    ${candidates_list}    Wait Until Keyword Succeeds    10s    1s    Get Cluster Entity Owner For Openflow    ${controller_index_list}
-    ...    openflow    openflow:1
+    ${owner}    ${candidates_list}    Wait Until Keyword Succeeds    10s    1s    ClusterOpenFlow.Get Cluster Entity Owner For Openflow    ${controller_index_list}
+    ...    openflow    ${device}

     [Return]    ${owner}    ${candidates_list}
 
     [Return]    ${owner}    ${candidates_list}
 

-Check OpenFlow Network Operational Information For One Device
+Check OpenFlow Network Operational Information For Sample Topology

     [Arguments]    ${controller_index_list}
     [Arguments]    ${controller_index_list}

-    [Documentation]    Check device openflow:1 is in operational inventory and topology in all instances in ${controller_index_list}.
-    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector.
-    ${dictionary}    Create Dictionary    openflow:1=4
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
-    ${dictionary}    Create Dictionary    openflow:1=11
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+    [Documentation]    Check devices in tree,2 are in operational inventory and topology in all instances in ${controller_index_list}.
+    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector
+    ...    + 5x node_id per link termination. TODO: A Keyword that can calculate this based on mininet topology.
+    ${dictionary}    Create Dictionary    openflow:1=4    openflow:2=5    openflow:3=5
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
+    ${dictionary}    Create Dictionary    openflow:1=21    openflow:2=19    openflow:3=19
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}

 
 Check No OpenFlow Network Operational Information
     [Arguments]    ${controller_index_list}
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances in ${controller_index_list}.
     ${dictionary}    Create Dictionary    openflow=0
 
 Check No OpenFlow Network Operational Information
     [Arguments]    ${controller_index_list}
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances in ${controller_index_list}.
     ${dictionary}    Create Dictionary    openflow=0

-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}

     ${dictionary}    Create Dictionary    openflow=0
     ${dictionary}    Create Dictionary    openflow=0

-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}

 
 Add Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}
 
 Add Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}


@@ -94,8 +95,8 @@ Add Sample Flow And Verify
     # replicating some of the matches in the flows section. Same comment applies for further keywords.
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"1"=1
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"1"=1
     # replicating some of the matches in the flows section. Same comment applies for further keywords.
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"1"=1
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"1"=1

-    Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}
-    Wait Until Keyword Succeeds    15s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    ClusterKeywords.Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}
+    Wait Until Keyword Succeeds    15s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}

 
 Modify Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}
 
 Modify Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}


@@ -103,15 +104,15 @@ Modify Sample Flow And Verify
     ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/sample_flow_2.json
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"2"=1
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"2"=1
     ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/sample_flow_2.json
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"2"=1
     Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"2"=1

-    Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}
-    Wait Until Keyword Succeeds    15s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    ClusterKeywords.Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}
+    Wait Until Keyword Succeeds    15s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}

 
 Delete Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Delete sample flow in Owner and verify it gets removed from all instances.
     ${dictionary}=    Create Dictionary    10.0.2.0/24=0
 
 Delete Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Delete sample flow in Owner and verify it gets removed from all instances.
     ${dictionary}=    Create Dictionary    10.0.2.0/24=0

-    Delete And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    ClusterKeywords.Delete And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}

 
 Send RPC Add Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}
 
 Send RPC Add Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}


@@ -122,7 +123,7 @@ Send RPC Add Sample Flow And Verify
     ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:add-flow    data=${body}    headers=${HEADERS_YANG_JSON}
     Log    ${resp.content}
     Should Be Equal As Strings    ${resp.status_code}    200
     ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:add-flow    data=${body}    headers=${HEADERS_YANG_JSON}
     Log    ${resp.content}
     Should Be Equal As Strings    ${resp.status_code}    200

-    Wait Until Keyword Succeeds    15s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    Wait Until Keyword Succeeds    15s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}

 
 Send RPC Delete Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}
 
 Send RPC Delete Sample Flow And Verify
     [Arguments]    ${controller_index_list}    ${controller_index}


@@ -132,18 +133,22 @@ Send RPC Delete Sample Flow And Verify
     ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:remove-flow    data=${body}    headers=${HEADERS_YANG_JSON}
     Log    ${resp.content}
     Should Be Equal As Strings    ${resp.status_code}    200
     ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:remove-flow    data=${body}    headers=${HEADERS_YANG_JSON}
     Log    ${resp.content}
     Should Be Equal As Strings    ${resp.status_code}    200

-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}

 
 Take OpenFlow Device Link Down and Verify
     [Arguments]    ${controller_index_list}
     [Documentation]    Take a link down and verify port status in all instances in ${controller_index_list}.
     ${dictionary}=    Create Dictionary    "link-down":true=1
 
 Take OpenFlow Device Link Down and Verify
     [Arguments]    ${controller_index_list}
     [Documentation]    Take a link down and verify port status in all instances in ${controller_index_list}.
     ${dictionary}=    Create Dictionary    "link-down":true=1

-    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 down
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+    ${ouput}=    MininetKeywords.Send Mininet Command    ${mininet_conn_id}    link s1 s2 down
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+    ${dictionary}    Create Dictionary    openflow:1=16    openflow:2=14    openflow:3=19
+    Wait Until Keyword Succeeds    20s    2s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}

 
 Take OpenFlow Device Link Up and Verify
     [Arguments]    ${controller_index_list}
     [Documentation]    Take the link up and verify port status in all instances in ${controller_index_list}.
     ${dictionary}=    Create Dictionary    "link-down":true=0
 
 Take OpenFlow Device Link Up and Verify
     [Arguments]    ${controller_index_list}
     [Documentation]    Take the link up and verify port status in all instances in ${controller_index_list}.
     ${dictionary}=    Create Dictionary    "link-down":true=0

-    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 up
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+    ${ouput}=    MininetKeywords.Send Mininet Command    ${mininet_conn_id}    link s1 s2 up
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+    ${dictionary}    Create Dictionary    openflow:1=21    openflow:2=19    openflow:3=19
+    Wait Until Keyword Succeeds    5s    1s    ClusterKeywords.Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}

diff --git a/csit/libraries/MininetKeywords.robot b/csit/libraries/MininetKeywords.robot
index ef3e44098545b654c389f293094e7c542cddec70..828fe3f66c5f7ba18d19d104ab4b6c30c43ecae9 100644 (file)
--- a/csit/libraries/MininetKeywords.robot
+++ b/csit/libraries/MininetKeywords.robot
@@ -2,6 +2,7 @@
 Documentation     Mininet library. This library is useful for tests using mininet tool to simulate devices.
 Library           SSHLibrary
 Resource          Utils.robot
 Documentation     Mininet library. This library is useful for tests using mininet tool to simulate devices.
 Library           SSHLibrary
 Resource          Utils.robot

+Resource          OVSDB.robot

 Variables         ../variables/Variables.py
 
 *** Keywords ***
 Variables         ../variables/Variables.py
 
 *** Keywords ***


@@ -9,15 +10,15 @@ Start Mininet Single Controller
     [Arguments]    ${mininet}=${TOOLS_SYSTEM_IP}    ${controller}=${ODL_SYSTEM_IP}    ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13    ${custom}=${EMPTY}    ${ofport}=6633
     [Documentation]    Start Mininet with custom topology and connect to controller.
     Log    Clear any existing mininet
     [Arguments]    ${mininet}=${TOOLS_SYSTEM_IP}    ${controller}=${ODL_SYSTEM_IP}    ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13    ${custom}=${EMPTY}    ${ofport}=6633
     [Documentation]    Start Mininet with custom topology and connect to controller.
     Log    Clear any existing mininet

-    Clean Mininet System    ${mininet}
-    ${mininet_conn_id}=    Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}
+    Utils.Clean Mininet System    ${mininet}
+    ${mininet_conn_id}=    SSHLibrary.Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}

     Set Suite Variable    ${mininet_conn_id}
     Set Suite Variable    ${mininet_conn_id}

-    Flexible Mininet Login
+    Utils.Flexible Mininet Login

     Run Keyword If    '${custom}' != '${EMPTY}'    Put File    ${custom}
     Log    Start mininet ${options} to ${controller}
     Run Keyword If    '${custom}' != '${EMPTY}'    Put File    ${custom}
     Log    Start mininet ${options} to ${controller}

-    Write    sudo mn --controller 'remote,ip=${controller},port=${ofport}' ${options}
-    Read Until    mininet>
-    ${output}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show
+    SSHLibrary.Write    sudo mn --controller 'remote,ip=${controller},port=${ofport}' ${options}
+    SSHLibrary.Read Until    mininet>
+    ${output}=    Utils.Run Command On Mininet    ${mininet}    sudo ovs-vsctl show

     Log    ${output}
     [Return]    ${mininet_conn_id}
 
     Log    ${output}
     [Return]    ${mininet_conn_id}
 


@@ -25,56 +26,37 @@ Start Mininet Multiple Controllers
     [Arguments]    ${mininet}    ${controller_index_list}    ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13    ${custom}=${EMPTY}    ${ofport}=6633
     [Documentation]    Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}.
     Log    Clear any existing mininet
     [Arguments]    ${mininet}    ${controller_index_list}    ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13    ${custom}=${EMPTY}    ${ofport}=6633
     [Documentation]    Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}.
     Log    Clear any existing mininet

-    Clean Mininet System    ${mininet}
-    ${mininet_conn_id}=    Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}
+    Utils.Clean Mininet System    ${mininet}
+    ${mininet_conn_id}=    SSHLibrary.Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}

     Set Suite Variable    ${mininet_conn_id}
     Set Suite Variable    ${mininet_conn_id}

-    Flexible Mininet Login
+    Utils.Flexible Mininet Login

     Run Keyword If    '${custom}' != '${EMPTY}'    Put File    ${custom}
     Log    Start mininet ${options}
     Run Keyword If    '${custom}' != '${EMPTY}'    Put File    ${custom}
     Log    Start mininet ${options}

-    Write    sudo mn ${options}
-    Read Until    mininet>
+    SSHLibrary.Write    sudo mn ${options}
+    SSHLibrary.Read Until    mininet>

     Log    Create controller configuration
     Log    Create controller configuration

-    ${ovs_opt}=    Set Variable
+    ${controller_opt}=    Set Variable

     : FOR    ${index}    IN    @{controller_index_list}
     : FOR    ${index}    IN    @{controller_index_list}

-    \    ${ovs_opt}=    Catenate    ${ovs_opt}    ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ofport}
-    \    Log    ${ovs_opt}
+    \    ${controller_opt}=    Catenate    ${controller_opt}    ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ofport}
+    \    Log    ${controller_opt}

     Log    Find Number of OVS bridges
     Log    Find Number of OVS bridges

-    ${num_bridges}    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show | grep Bridge | wc -l
+    ${num_bridges}    Utils.Run Command On Mininet    ${mininet}    sudo ovs-vsctl show | grep Bridge | wc -l

     ${num_bridges}=    Convert To Integer    ${num_bridges}
     ${num_bridges}=    Convert To Integer    ${num_bridges}

-    Log    Configure OVS controllers ${ovs_opt} in all bridges
+    Log    Configure OVS controllers ${controller_opt} in all bridges

     : FOR    ${i}    IN RANGE    1    ${num_bridges+1}
     : FOR    ${i}    IN RANGE    1    ${num_bridges+1}

-    \    ${bridge}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show | grep Bridge | cut -c 12- | sort | head -${i} | tail -1
-    \    Run Command On Mininet    ${mininet}    sudo ovs-vsctl set-controller ${bridge} ${ovs_opt}
+    \    ${bridge}=    Utils.Run Command On Mininet    ${mininet}    sudo ovs-vsctl show | grep Bridge | cut -c 12- | sort | head -${i} | tail -1
+    \    OVSDB.Set Controller In OVS Bridge    ${mininet}    ${bridge}    ${controller_opt}

     Log    Check OVS configuratiom
     Log    Check OVS configuratiom

-    ${output}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show
-    Log    ${output}
-    [Return]    ${mininet_conn_id}
-
-Add Multiple Managers to OVS
-    [Arguments]    ${mininet}    ${controller_index_list}    ${ovs_mgr_port}=6640
-    [Documentation]    Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}.
-    Log    Clear any existing mininet
-    Clean Mininet System    ${mininet}
-    ${mininet_conn_id}=    Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}
-    Set Suite Variable    ${mininet_conn_id}
-    Flexible Mininet Login
-    ${ovs_opt}=    Set Variable
-    : FOR    ${index}    IN    @{controller_index_list}
-    \    ${ovs_opt}=    Catenate    ${ovs_opt}    ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ovs_mgr_port}
-    \    Log    ${ovs_opt}
-    Log    Configure OVS Managers in the OVS
-    Run Command On Mininet    ${mininet}    sudo ovs-vsctl set-manager ${ovs_opt}
-    Log    Check OVS configuratiom
-    ${output}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show
+    ${output}=    Utils.Run Command On Mininet    ${mininet}    sudo ovs-vsctl show

     Log    ${output}
     [Return]    ${mininet_conn_id}
 
 Send Mininet Command
     [Arguments]    ${mininet_conn_id}    ${cmd}=help
     [Documentation]    Sends Command ${cmd} to Mininet session ${mininet_conn_id} and returns read buffer response.
     Log    ${output}
     [Return]    ${mininet_conn_id}
 
 Send Mininet Command
     [Arguments]    ${mininet_conn_id}    ${cmd}=help
     [Documentation]    Sends Command ${cmd} to Mininet session ${mininet_conn_id} and returns read buffer response.

-    Switch Connection    ${mininet_conn_id}
+    SSHLibrary.Switch Connection    ${mininet_conn_id}

     SSHLibrary.Write    ${cmd}
     SSHLibrary.Write    ${cmd}

-    ${output}=    Read Until    mininet>
+    ${output}=    SSHLibrary.Read Until    mininet>

     [Return]    ${output}
 
 Send Mininet Command Multiple Sessions
     [Return]    ${output}
 
 Send Mininet Command Multiple Sessions


@@ -82,20 +64,20 @@ Send Mininet Command Multiple Sessions
     [Documentation]    Sends Command ${cmd} to Mininet sessions in ${mininet_conn_list} and returns list of read buffer responses.
     ${output_list}=    Create List
     : FOR    ${mininet_conn_id}    IN    @{mininet_conn_list}
     [Documentation]    Sends Command ${cmd} to Mininet sessions in ${mininet_conn_list} and returns list of read buffer responses.
     ${output_list}=    Create List
     : FOR    ${mininet_conn_id}    IN    @{mininet_conn_list}

-    \    ${output}=    Send Mininet Command    ${mininet_conn_id}    ${cmd}
+    \    ${output}=    Utils.Send Mininet Command    ${mininet_conn_id}    ${cmd}

     \    Append To List    ${output_list}    ${output}
     [Return]    ${output_list}
 
 Stop Mininet And Exit
     [Arguments]    ${mininet_conn_id}
     [Documentation]    Stops Mininet and exits session ${mininet_conn_id}
     \    Append To List    ${output_list}    ${output}
     [Return]    ${output_list}
 
 Stop Mininet And Exit
     [Arguments]    ${mininet_conn_id}
     [Documentation]    Stops Mininet and exits session ${mininet_conn_id}

-    Switch Connection    ${mininet_conn_id}
+    SSHLibrary.Switch Connection    ${mininet_conn_id}

     SSHLibrary.Write    exit
     SSHLibrary.Write    exit

-    Read Until    ${TOOLS_SYSTEM_PROMPT}
+    SSHLibrary.Read Until    ${TOOLS_SYSTEM_PROMPT}

     Close Connection
 
 Stop Mininet And Exit Multiple Sessions
     [Arguments]    ${mininet_conn_list}
     [Documentation]    Stops Mininet and exits sessions in ${mininet_conn_list}.
     : FOR    ${mininet_conn_id}    IN    @{mininet_conn_list}
     Close Connection
 
 Stop Mininet And Exit Multiple Sessions
     [Arguments]    ${mininet_conn_list}
     [Documentation]    Stops Mininet and exits sessions in ${mininet_conn_list}.
     : FOR    ${mininet_conn_id}    IN    @{mininet_conn_list}

-    \    Stop Mininet And Exit    ${mininet_conn_id}
+    \    MininetKeywords.Stop Mininet And Exit    ${mininet_conn_id}

diff --git a/csit/libraries/OVSDB.robot b/csit/libraries/OVSDB.robot
index ec50a5a8f7d17b41b74165a562fae968a13170eb..d4b4fb3994cba367d9dec3b4172abe126561d055 100644 (file)
--- a/csit/libraries/OVSDB.robot
+++ b/csit/libraries/OVSDB.robot
@@ -109,3 +109,29 @@ Clean OVSDB Test Environment
     Run Command On Mininet    ${tools_system}    sudo /usr/share/openvswitch/scripts/ovs-ctl stop
     Run Command On Mininet    ${tools_system}    sudo rm -rf /etc/openvswitch/conf.db
     Run Command On Mininet    ${tools_system}    sudo /usr/share/openvswitch/scripts/ovs-ctl start
     Run Command On Mininet    ${tools_system}    sudo /usr/share/openvswitch/scripts/ovs-ctl stop
     Run Command On Mininet    ${tools_system}    sudo rm -rf /etc/openvswitch/conf.db
     Run Command On Mininet    ${tools_system}    sudo /usr/share/openvswitch/scripts/ovs-ctl start

+
+Set Controller In OVS Bridge
+    [Arguments]    ${mininet}    ${bridge}    ${controller_opt}
+    [Documentation]    Sets controller for a given OVS ${bridge} using controller options in ${controller_opt}
+    Run Command On Mininet    ${mininet}    sudo ovs-vsctl del-controller ${bridge}
+    Run Command On Mininet    ${mininet}    sudo ovs-vsctl set-controller ${bridge} ${controller_opt}
+
+Add Multiple Managers to OVS
+    [Arguments]    ${mininet}    ${controller_index_list}    ${ovs_mgr_port}=6640
+    [Documentation]    Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}.
+    Log    Clear any existing mininet
+    Clean Mininet System    ${mininet}
+    ${mininet_conn_id}=    Open Connection    ${mininet}    prompt=${TOOLS_SYSTEM_PROMPT}    timeout=${DEFAULT_TIMEOUT}
+    Set Suite Variable    ${mininet_conn_id}
+    Flexible Mininet Login
+    ${ovs_opt}=    Set Variable
+    : FOR    ${index}    IN    @{controller_index_list}
+    \    ${ovs_opt}=    Catenate    ${ovs_opt}    ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ovs_mgr_port}
+    \    Log    ${ovs_opt}
+    Log    Configure OVS Managers in the OVS
+    Run Command On Mininet    ${mininet}    sudo ovs-vsctl set-manager ${ovs_opt}
+    Log    Check OVS configuratiom
+    ${output}=    Run Command On Mininet    ${mininet}    sudo ovs-vsctl show
+    Log    ${output}
+    [Return]    ${mininet_conn_id}
+

diff --git a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
index 3454295405ebdc3c567dee95567f47eb41e15177..29174e4c8a967520f023f581cb4d393182ec3833 100644 (file)
--- a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
+++ b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
@@ -11,220 +11,230 @@ Variables         ../../../variables/Variables.py
 *** Test Cases ***
 Create Original Cluster List
     [Documentation]    Create original cluster list.
 *** Test Cases ***
 Create Original Cluster List
     [Documentation]    Create original cluster list.

-    ${original_cluster_list}    Create Controller Index List
+    ${original_cluster_list}    ClusterKeywords.Create Controller Index List

     Set Suite Variable    ${original_cluster_list}
 
 Check Shards Status Before Fail
     [Documentation]    Check Status for all shards in OpenFlow application.
     Set Suite Variable    ${original_cluster_list}
 
 Check Shards Status Before Fail
     [Documentation]    Check Status for all shards in OpenFlow application.

-    Check OpenFlow Shards Status    ${original_cluster_list}
+    ClusterOpenFlow.Check OpenFlow Shards Status    ${original_cluster_list}

 
 Start Mininet Multiple Connections
 
 Start Mininet Multiple Connections

-    [Documentation]    Start mininet with connection to all cluster instances.
-    ${mininet_conn_id}=    Start Mininet Multiple Controllers    ${TOOLS_SYSTEM_IP}    ${original_cluster_list}
+    [Documentation]    Start mininet tree,2 with connection to all cluster instances.
+    ${mininet_conn_id}=    MininetKeywords.Start Mininet Multiple Controllers    ${TOOLS_SYSTEM_IP}    ${original_cluster_list}    --topo tree,2 --switch ovsk,protocols=OpenFlow13

     Set Suite Variable    ${mininet_conn_id}
 
 Check Entity Owner Status And Find Owner and Candidate Before Fail
     Set Suite Variable    ${mininet_conn_id}
 
 Check Entity Owner Status And Find Owner and Candidate Before Fail

-    [Documentation]    Check Entity Owner Status and identify owner and candidate.
-    ${original_owner}    ${original_candidates_list}    Get OpenFlow Entity Owner Status For One Device    ${original_cluster_list}
+    [Documentation]    Check Entity Owner Status and identify owner and candidate for first switch s1.
+    ${original_owner}    ${original_candidates_list}    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    ${original_cluster_list}    openflow:1

     ${original_candidate}=    Get From List    ${original_candidates_list}    0
     Set Suite Variable    ${original_owner}
     Set Suite Variable    ${original_candidate}
 
     ${original_candidate}=    Get From List    ${original_candidates_list}    0
     Set Suite Variable    ${original_owner}
     Set Suite Variable    ${original_candidate}
 

+Reconnect Extra Switches To Candidate And Check Entity Owner
+    [Documentation]    Connect switches s2 and s3 to candidate instance.
+    OVSDB.Set Controller In OVS Bridge    ${TOOLS_SYSTEM_IP}    s2    tcp:${ODL_SYSTEM_${original_candidate}_IP}:6633
+    OVSDB.Set Controller In OVS Bridge    ${TOOLS_SYSTEM_IP}    s3    tcp:${ODL_SYSTEM_${original_candidate}_IP}:6633
+    ${original_candidate_list}=    Create List    ${original_candidate}
+    ${owner}    ${candidates_list}    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    ${original_candidate_list}    openflow:2
+    Should Be Equal    ${owner}    ${original_candidate}
+    ${owner}    ${candidates_list}    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    ${original_candidate_list}    openflow:3
+    Should Be Equal    ${owner}    ${original_candidate}
+

 Check Network Operational Information Before Fail
 Check Network Operational Information Before Fail

-    [Documentation]    Check device is in operational inventory and topology in all cluster instances.
-    Check OpenFlow Network Operational Information For One Device    ${original_cluster_list}
+    [Documentation]    Check devices in operational inventory and topology in all cluster instances.
+    ClusterOpenFlow.Check OpenFlow Network Operational Information For Sample Topology    ${original_cluster_list}

 
 Add Configuration In Owner and Verify Before Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Add Configuration In Owner and Verify Before Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Modify Configuration In Owner and Verify Before Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
 
 Modify Configuration In Owner and Verify Before Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    Modify Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Modify Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Delete Configuration In Owner and Verify Before Fail
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
 
 Delete Configuration In Owner and Verify Before Fail
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.

-    Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Add Configuration In Candidate and Verify Before Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Add Configuration In Candidate and Verify Before Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Add Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+    ClusterOpenFlow.Add Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 Modify Configuration In Candidate and Verify Before Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
 
 Modify Configuration In Candidate and Verify Before Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    Modify Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+    ClusterOpenFlow.Modify Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 Delete Configuration In Candidate and Verify Before Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
 
 Delete Configuration In Candidate and Verify Before Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    Delete Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+    ClusterOpenFlow.Delete Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 Send RPC Add to Owner and Verify Before Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Send RPC Add to Owner and Verify Before Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Send RPC Delete to Owner and Verify Before Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
 
 Send RPC Delete to Owner and Verify Before Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Send RPC Add to Candidate and Verify Before Fail
     [Documentation]    Add Flow in Candidate and verify it gets applied from all instances.
 
 Send RPC Add to Candidate and Verify Before Fail
     [Documentation]    Add Flow in Candidate and verify it gets applied from all instances.

-    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+    ClusterOpenFlow.Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 Send RPC Delete to Candidate and Verify Before Fail
     [Documentation]    Delete Flow in Candidate and verify it gets removed from all instances.
 
 Send RPC Delete to Candidate and Verify Before Fail
     [Documentation]    Delete Flow in Candidate and verify it gets removed from all instances.

-    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+    ClusterOpenFlow.Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 Modify Network And Verify Before Fail
     [Documentation]    Take a link down and verify port status in all instances.
 
 Modify Network And Verify Before Fail
     [Documentation]    Take a link down and verify port status in all instances.

-    Take OpenFlow Device Link Down and Verify    ${original_cluster_list}
+    ClusterOpenFlow.Take OpenFlow Device Link Down and Verify    ${original_cluster_list}

 
 Restore Network And Verify Before Fail
     [Documentation]    Take the link up and verify port status in all instances.
 
 Restore Network And Verify Before Fail
     [Documentation]    Take the link up and verify port status in all instances.

-    Take OpenFlow Device Link Up and Verify    ${original_cluster_list}
+    ClusterOpenFlow.Take OpenFlow Device Link Up and Verify    ${original_cluster_list}

 
 Kill Owner Instance
     [Documentation]    Kill Owner Instance and verify it is dead
 
 Kill Owner Instance
     [Documentation]    Kill Owner Instance and verify it is dead

-    Kill Multiple Controllers    ${original_owner}
+    ClusterKeywords.Kill Multiple Controllers    ${original_owner}

     ${new_cluster_list}    Create Controller Index List
     Remove Values From List    ${new_cluster_list}    ${original_owner}
     Set Suite Variable    ${new_cluster_list}
 
 Check Shards Status After Fail
     [Documentation]    Create original cluster list and check Status for all shards in OpenFlow application.
     ${new_cluster_list}    Create Controller Index List
     Remove Values From List    ${new_cluster_list}    ${original_owner}
     Set Suite Variable    ${new_cluster_list}
 
 Check Shards Status After Fail
     [Documentation]    Create original cluster list and check Status for all shards in OpenFlow application.

-    Check OpenFlow Shards Status After Cluster Event    ${new_cluster_list}
+    ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${new_cluster_list}

 
 Check Entity Owner Status And Find Owner and Candidate After Fail
     [Documentation]    Check Entity Owner Status and identify owner and candidate.
 
 Check Entity Owner Status And Find Owner and Candidate After Fail
     [Documentation]    Check Entity Owner Status and identify owner and candidate.

-    ${new_owner}    ${new_candidates_list}    Get OpenFlow Entity Owner Status For One Device    ${new_cluster_list}
+    ${new_owner}    ${new_candidates_list}    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    ${new_cluster_list}    openflow:1

     Run Keyword And Continue On Failure    List Should Not Contain Value    ${new_candidates_list}    ${original_owner}    Original owner ${original_owner} still in candidate list.
     Remove Values From List    ${new_candidates_list}    ${original_owner}
     ${new_candidate}=    Get From List    ${new_candidates_list}    0
     Set Suite Variable    ${new_owner}
     Set Suite Variable    ${new_candidate}
     Run Keyword And Continue On Failure    List Should Not Contain Value    ${new_candidates_list}    ${original_owner}    Original owner ${original_owner} still in candidate list.
     Remove Values From List    ${new_candidates_list}    ${original_owner}
     ${new_candidate}=    Get From List    ${new_candidates_list}    0
     Set Suite Variable    ${new_owner}
     Set Suite Variable    ${new_candidate}

-    [Teardown]    Report_Failure_Due_To_Bug    5004
+    [Teardown]    Utils.Report_Failure_Due_To_Bug    5004

 
 Check Network Operational Information After Fail
 
 Check Network Operational Information After Fail

-    [Documentation]    Check device is in operational inventory and topology in all cluster instances.
-    Check OpenFlow Network Operational Information For One Device    ${new_cluster_list}
+    [Documentation]    Check devices in operational inventory and topology in all cluster instances.
+    ClusterOpenFlow.Check OpenFlow Network Operational Information For Sample Topology    ${new_cluster_list}

 
 Add Configuration In Owner and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Add Configuration In Owner and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Add Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Add Sample Flow and Verify    ${new_cluster_list}    ${new_owner}

 
 Modify Configuration In Owner and Verify After Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
 
 Modify Configuration In Owner and Verify After Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    Modify Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Modify Sample Flow and Verify    ${new_cluster_list}    ${new_owner}

 
 Delete Configuration In Owner and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
 
 Delete Configuration In Owner and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.

-    Delete Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Delete Sample Flow and Verify    ${new_cluster_list}    ${new_owner}

 
 Add Configuration In Candidate and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Add Configuration In Candidate and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Add Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+    ClusterOpenFlow.Add Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}

 
 Modify Configuration In Candidate and Verify After Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
 
 Modify Configuration In Candidate and Verify After Fail
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    Modify Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+    ClusterOpenFlow.Modify Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}

 
 Delete Configuration In Candidate and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
 
 Delete Configuration In Candidate and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.

-    Delete Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+    ClusterOpenFlow.Delete Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}

 
 Send RPC Add to Owner and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Send RPC Add to Owner and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Send RPC Add Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Send RPC Add Sample Flow and Verify    ${new_cluster_list}    ${new_owner}

 
 Send RPC Delete to Owner and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
 
 Send RPC Delete to Owner and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    Send RPC Delete Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Send RPC Delete Sample Flow and Verify    ${new_cluster_list}    ${new_owner}

 
 Send RPC Add to Candidate and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Send RPC Add to Candidate and Verify After Fail
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Send RPC Add Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+    ClusterOpenFlow.Send RPC Add Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}

 
 Send RPC Delete to Candidate and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
 
 Send RPC Delete to Candidate and Verify After Fail
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    Send RPC Delete Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+    ClusterOpenFlow.Send RPC Delete Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}

 
 Modify Network and Verify After Fail
     [Documentation]    Take a link down and verify port status in all instances.
 
 Modify Network and Verify After Fail
     [Documentation]    Take a link down and verify port status in all instances.

-    Take OpenFlow Device Link Down and Verify    ${new_cluster_list}
+    ClusterOpenFlow.Take OpenFlow Device Link Down and Verify    ${new_cluster_list}

 
 Restore Network and Verify After Fail
     [Documentation]    Take the link up and verify port status in all instances.
 
 Restore Network and Verify After Fail
     [Documentation]    Take the link up and verify port status in all instances.

-    Take OpenFlow Device Link Up and Verify    ${new_cluster_list}
+    ClusterOpenFlow.Take OpenFlow Device Link Up and Verify    ${new_cluster_list}

 
 Start Old Owner Instance
     [Documentation]    Start old Owner Instance and verify it is up
 
 Start Old Owner Instance
     [Documentation]    Start old Owner Instance and verify it is up

-    Start Multiple Controllers    300s    ${original_owner}
+    ClusterKeywords.Start Multiple Controllers    300s    ${original_owner}

 
 Check Shards Status After Recover
     [Documentation]    Create original cluster list and check Status for all shards in OpenFlow application.
 
 Check Shards Status After Recover
     [Documentation]    Create original cluster list and check Status for all shards in OpenFlow application.

-    Check OpenFlow Shards Status After Cluster Event    ${original_cluster_list}
+    ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event    ${original_cluster_list}

 
 Check Entity Owner Status After Recover
     [Documentation]    Check Entity Owner Status and identify owner and candidate.
 
 Check Entity Owner Status After Recover
     [Documentation]    Check Entity Owner Status and identify owner and candidate.

-    ${new_owner}    ${new_candidates_list}    Get OpenFlow Entity Owner Status For One Device    ${original_cluster_list}
+    ${new_owner}    ${new_candidates_list}    ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device    ${original_cluster_list}    openflow:1

     Set Suite Variable    ${new_owner}
 
 Check Network Operational Information After Recover
     Set Suite Variable    ${new_owner}
 
 Check Network Operational Information After Recover

-    [Documentation]    Check device is in operational inventory and topology in all cluster instances.
-    Check OpenFlow Network Operational Information For One Device    ${original_cluster_list}
+    [Documentation]    Check devices in operational inventory and topology in all cluster instances.
+    ClusterOpenFlow.Check OpenFlow Network Operational Information For Sample Topology    ${original_cluster_list}

 
 Add Configuration In Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Add Configuration In Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Add Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Add Sample Flow and Verify    ${original_cluster_list}    ${new_owner}

 
 Modify Configuration In Owner and Verify After Recover
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
 
 Modify Configuration In Owner and Verify After Recover
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    Modify Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Modify Sample Flow and Verify    ${original_cluster_list}    ${new_owner}

 
 Delete Configuration In Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
 
 Delete Configuration In Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.

-    Delete Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Delete Sample Flow and Verify    ${original_cluster_list}    ${new_owner}

 
 Add Configuration In Old Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Add Configuration In Old Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Add Sample Flow and Verify    ${originalcluster_list}    ${original_owner}
+    ClusterOpenFlow.Add Sample Flow and Verify    ${originalcluster_list}    ${original_owner}

 
 Modify Configuration In Old Owner and Verify After Recover
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
 
 Modify Configuration In Old Owner and Verify After Recover
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    Modify Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Modify Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Delete Configuration In Old Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
 
 Delete Configuration In Old Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.

-    Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Send RPC Add to Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Send RPC Add to Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${new_owner}

 
 Send RPC Delete to Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
 
 Send RPC Delete to Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+    ClusterOpenFlow.Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${new_owner}

 
 Send RPC Add to Old Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
 
 Send RPC Add to Old Owner and Verify After Recover
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Send RPC Delete to Old Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
 
 Send RPC Delete to Old Owner and Verify After Recover
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+    ClusterOpenFlow.Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 Modify Network and Verify After Recover
     [Documentation]    Take a link down and verify port status in all instances.
 
 Modify Network and Verify After Recover
     [Documentation]    Take a link down and verify port status in all instances.

-    Take OpenFlow Device Link Down and Verify    ${original_cluster_list}
+    ClusterOpenFlow.Take OpenFlow Device Link Down and Verify    ${original_cluster_list}

 
 Restore Network and Verify After Recover
     [Documentation]    Take the link up and verify port status in all instances.
 
 Restore Network and Verify After Recover
     [Documentation]    Take the link up and verify port status in all instances.

-    Take OpenFlow Device Link Up and Verify    ${original_cluster_list}
+    ClusterOpenFlow.Take OpenFlow Device Link Up and Verify    ${original_cluster_list}

 
 Stop Mininet and Exit
     [Documentation]    Stop mininet and exit connection.
 
 Stop Mininet and Exit
     [Documentation]    Stop mininet and exit connection.

-    Stop Mininet And Exit    ${mininet_conn_id}
-    Clean Mininet System
+    MininetKeywords.Stop Mininet And Exit    ${mininet_conn_id}
+    Utils.Clean Mininet System

 
 Check No Network Operational Information
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances.
 
 Check No Network Operational Information
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances.

-    Check No OpenFlow Network Operational Information    ${original_cluster_list}
+    ClusterOpenFlow.Check No OpenFlow Network Operational Information    ${original_cluster_list}