X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=csit%2Flibraries%2FClusterOpenFlow.robot;h=0bc1a4a27eede2e06da819f278b933ced9807b21;hb=2d75130cbe2149eac8a8b874368e9719b0c6441d;hp=0a3fd8043778a0f8a093c9c2b66af8ee32136de0;hpb=cced0335a8a98e4e680eba0a11417d4f59c8c520;p=integration%2Ftest.git

diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot
index 0a3fd80437..0bc1a4a27e 100644
--- a/csit/libraries/ClusterOpenFlow.robot
+++ b/csit/libraries/ClusterOpenFlow.robot
@@ -1,147 +1,370 @@
 *** Settings ***
-Documentation     Cluster OpenFlow library. So far this library is only to be used by OpenFlow cluster test as it is very specific for this test.
-Library           RequestsLibrary
-Resource          ClusterKeywords.robot
-Resource          MininetKeywords.robot
-Resource          Utils.robot
-Variables         ../variables/Variables.py
+Documentation       Cluster OpenFlow library. So far this library is only to be used by OpenFlow cluster test as it is very specific for this test.
+
+Library             Collections
+Library             RequestsLibrary
+Library             ${CURDIR}/ScaleClient.py
+Resource            ClusterManagement.robot
+Resource            CompareStream.robot
+Resource            MininetKeywords.robot
+Resource            Utils.robot
+Resource            ../variables/openflowplugin/Variables.robot
+Variables           ../variables/Variables.py
+
 
 *** Variables ***
-${config_table_0}    ${CONFIG_NODES_API}/node/openflow:1/table/0
-${operational_table_0}    ${OPERATIONAL_NODES_API}/node/openflow:1/table/0
-${operational_port_1}    ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector/openflow:1:1
+@{SHARD_OPER_LIST}          inventory    topology    default    entity-ownership
+@{SHARD_CONF_LIST}          inventory    topology    default
+${config_table_0}           ${RFC8040_NODES_API}/node=openflow%3A1/flow-node-inventory:table=0
+${operational_table_0}
+...                         ${RFC8040_NODES_API}/node=openflow%3A1/flow-node-inventory:table=0?${RFC8040_OPERATIONAL_CONTENT}
+${operational_port_1}
+...                         ${RFC8040_NODES_API}/node=openflow%3A1/node-connector=openflow%3A1%3A1?${RFC8040_OPERATIONAL_CONTENT}
+
 
 *** Keywords ***
+Get InventoryConfig Shard Status
+    [Documentation]    Check Status for Inventory Config shard in OpenFlow application.
+    [Arguments]    ${controller_index_list}=${EMPTY}
+    ${inv_conf_leader}    ${inv_conf_followers_list}=    Wait Until Keyword Succeeds
+    ...    10s
+    ...    1s
+    ...    ClusterManagement.Get_Leader_And_Followers_For_Shard
+    ...    shard_name=inventory
+    ...    shard_type=config
+    ...    member_index_list=${controller_index_list}
+    Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
+    RETURN    ${inv_conf_leader}    ${inv_conf_followers_list}
+
 Check OpenFlow Shards Status
-    [Arguments]    ${controller_index_list}
     [Documentation]    Check Status for all shards in OpenFlow application.
-    ${inv_conf_leader}    ${inv_conf_followers_list}    Get Cluster Shard Status    ${controller_index_list}    config    inventory
-    ${inv_oper_leader}    ${inv_oper_followers_list}    Get Cluster Shard Status    ${controller_index_list}    operational    inventory
-    ${topo_oper_leader}    ${topo_oper_followers_list}    Get Cluster Shard Status    ${controller_index_list}    operational    topology
-    ${owner_oper_leader}    ${owner_oper_followers_list}    Get Cluster Shard Status    ${controller_index_list}    operational    entity-ownership
-    Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
-    Log    operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list}
-    Log    operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list}
-    Log    operational entity-ownership Leader is ${owner_oper_leader} and followers are ${owner_oper_followers_list}
+    [Arguments]    ${controller_index_list}=${EMPTY}
+    CompareStream.Run_Keyword_If_At_Least_Phosphorus
+    ...    Collections.Remove Values From List
+    ...    ${SHARD_OPER_LIST}
+    ...    entity-ownership
+    Log    ${SHARD_OPER_LIST}
+    ClusterManagement.Verify_Leader_Exists_For_Each_Shard
+    ...    shard_name_list=${SHARD_OPER_LIST}
+    ...    shard_type=operational
+    ...    member_index_list=${controller_index_list}
+    ClusterManagement.Verify_Leader_Exists_For_Each_Shard
+    ...    shard_name_list=${SHARD_CONF_LIST}
+    ...    shard_type=config
+    ...    member_index_list=${controller_index_list}
 
 Check OpenFlow Shards Status After Cluster Event
-    [Arguments]    ${controller_index_list}
     [Documentation]    Check Shards Status after some cluster event.
-    Wait Until Keyword Succeeds    90s    1s    Check OpenFlow Shards Status    ${controller_index_list}
-
-Get Cluster Entity Owner For Openflow
-    [Arguments]    ${controller_index_list}    ${device_type}    ${device}
-    [Documentation]    Checks OpenFlow Entity Owner status for a ${device} and returns owner index and list of candidates from a ${controller_index_list}.
-    ...    ${device_type} is normally openflow.
-    ${length}=    Get Length    ${controller_index_list}
-    ${candidates_list}=    Create List
-    ${data}=    Get Data From URI    controller@{controller_index_list}[0]    /restconf/operational/entity-owners:entity-owners
-    Log    ${data}
-    ${data}=    Replace String    ${data}    /general-entity:entity[general-entity:name='    ${EMPTY}
-    ${clear_data}=    Replace String    ${data}    ']    ${EMPTY}
-    Log    ${clear_data}
-    ${json}=    To Json    ${clear_data}
-    ${entity_type_list}=    Get From Dictionary    &{json}[entity-owners]    entity-type
-    ${entity_type_index}=    Get Index From List Of Dictionaries    ${entity_type_list}    type    ${device_type}
-    Should Not Be Equal    ${entity_type_index}    -1    No Entity Owner found for ${device_type}
-    ${entity_list}=    Get From Dictionary    @{entity_type_list}[${entity_type_index}]    entity
-    ${entity_index}=    Get Index From List Of Dictionaries    ${entity_list}    id    ${device}
-    Should Not Be Equal    ${entity_index}    -1    Device ${device} not found in Entity Owner ${device_type}
-    ${entity_owner}=    Get From Dictionary    @{entity_list}[${entity_index}]    owner
-    Should Not Be Empty    ${entity_owner}    No owner found for ${device}
-    ${owner}=    Replace String    ${entity_owner}    member-    ${EMPTY}
-    ${owner}=    Convert To Integer    ${owner}
-    List Should Contain Value    ${controller_index_list}    ${owner}    Owner ${owner} not exisiting in ${controller_index_list}
-    ${entity_candidates_list}=    Get From Dictionary    @{entity_list}[${entity_index}]    candidate
-    ${list_length}=    Get Length    ${entity_candidates_list}
-    : FOR    ${entity_candidate}    IN    @{entity_candidates_list}
-    \    ${candidate}=    Replace String    &{entity_candidate}[name]    member-    ${EMPTY}
-    \    ${candidate}=    Convert To Integer    ${candidate}
-    \    Run Keyword If    '${candidate}' != '${owner}'    Append To List    ${candidates_list}    ${candidate}
-    [Return]    ${owner}    ${candidates_list}
+    [Arguments]    ${controller_index_list}=${EMPTY}
+    Wait Until Keyword Succeeds
+    ...    90s
+    ...    1s
+    ...    ClusterOpenFlow.Check OpenFlow Shards Status
+    ...    ${controller_index_list}
 
 Get OpenFlow Entity Owner Status For One Device
-    [Arguments]    ${controller_index_list}
-    [Documentation]    Check Entity Owner Status and identify owner and candidate.
-    ${owner}    ${candidates_list}    Wait Until Keyword Succeeds    10s    1s    Get Cluster Entity Owner For Openflow    ${controller_index_list}
-    ...    openflow    openflow:1
-    [Return]    ${owner}    ${candidates_list}
-
-Check OpenFlow Network Operational Information For One Device
-    [Arguments]    ${controller_index_list}
-    [Documentation]    Check device openflow:1 is in operational inventory and topology in all instances in ${controller_index_list}.
-    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector.
-    ${dictionary}    Create Dictionary    openflow:1=4
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
-    ${dictionary}    Create Dictionary    openflow:1=11
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+    [Documentation]    Check Entity Owner Status and identify owner and successors for the device ${device}. Request is sent to controller ${controller_index}.
+    [Arguments]    ${device}    ${controller_index}    ${controller_index_list}=${EMPTY}    ${after_stop}=False
+    ${owner}    ${successor_list}=    Wait Until Keyword Succeeds
+    ...    30s
+    ...    1s
+    ...    ClusterManagement.Verify_Owner_And_Successors_For_Device
+    ...    device_name=${device}
+    ...    device_type=openflow
+    ...    member_index=${controller_index}
+    ...    candidate_list=${controller_index_list}
+    ...    after_stop=${after_stop}
+    RETURN    ${owner}    ${successor_list}
+
+Check OpenFlow Device Owner
+    [Documentation]    Check owner and candidates for the device ${device}. Request is sent to controller ${controller_index}.
+    [Arguments]    ${device}    ${controller_index}    ${expected_owner}    ${expected_candidate_list}=${EMPTY}
+    ${owner}    ${successor_list}=    ClusterManagement.Verify_Owner_And_Successors_For_Device
+    ...    device_name=${device}
+    ...    device_type=openflow
+    ...    member_index=${controller_index}
+    ...    candidate_list=${expected_candidate_list}
+    Should Be Equal    ${owner}    ${expected_owner}
+
+Check OpenFlow Network Operational Information For Sample Topology
+    [Documentation]    Check devices in tree,2 are in operational inventory and topology in all instances in ${controller_index_list}.
+    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector
+    ...    + 5x node_id per link termination. TODO: A Keyword that can calculate this based on mininet topology.
+    [Arguments]    ${controller_index_list}=${EMPTY}
+    ${dictionary}=    Create Dictionary    openflow:1=4    openflow:2=5    openflow:3=5
+    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_NODES_API}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
+    ${dictionary}=    Create Dictionary    openflow:1=21    openflow:2=19    openflow:3=19
+    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_TOPO_API}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Check No OpenFlow Network Operational Information
-    [Arguments]    ${controller_index_list}
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances in ${controller_index_list}.
-    ${dictionary}    Create Dictionary    openflow=0
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
-    ${dictionary}    Create Dictionary    openflow=0
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+    [Arguments]    ${controller_index_list}=${EMPTY}
+    ${dictionary}=    Create Dictionary    openflow=0
+    CompareStream.Run_Keyword_If_At_Least_Neon
+    ...    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_No_Content_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_NODES_API}
+    ...    member_index_list=${controller_index_list}
+    CompareStream.Run_Keyword_If_At_Most_Fluorine
+    ...    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_NODES_API}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
+    ${dictionary}=    Create Dictionary    openflow=0
+    Wait Until Keyword Succeeds
+    ...    20s
+    ...    2s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_TOPO_API}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Add Sample Flow And Verify
-    [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Add sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index}    ${controller_index_list}=${EMPTY}
     ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/sample_flow_1.json
     # There are slight differences on the way He and Li plugin display table information. He plugin has an additional Hashmap field
     # replicating some of the matches in the flows section. Same comment applies for further keywords.
-    Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"1"=1
-    Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"1"=1
-    Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}
-    Wait Until Keyword Succeeds    15s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    IF    '${ODL_OF_PLUGIN}' == 'helium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"1"=1
+    END
+    IF    '${ODL_OF_PLUGIN}' == 'lithium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"1"=1
+    END
+    ClusterManagement.Put_As_Json_And_Check_Member_List_Or_All
+    ...    ${config_table_0}/flow=1
+    ...    ${body}
+    ...    ${controller_index}
+    ...    ${controller_index_list}
+    Wait Until Keyword Succeeds
+    ...    15s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_table_0}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
+
+Verify Sample Flow
+    [Documentation]    Verify sample flow gets applied in all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index_list}=${EMPTY}
+    # There are slight differences on the way He and Li plugin display table information. He plugin has an additional Hashmap field
+    # replicating some of the matches in the flows section. Same comment applies for further keywords.
+    IF    '${ODL_OF_PLUGIN}' == 'helium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"1"=1
+    END
+    IF    '${ODL_OF_PLUGIN}' == 'lithium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"1"=1
+    END
+    Wait Until Keyword Succeeds
+    ...    15s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_table_0}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Modify Sample Flow And Verify
-    [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Modify sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index}    ${controller_index_list}=${EMPTY}
     ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/sample_flow_2.json
-    Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"2"=1
-    Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"2"=1
-    Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}
-    Wait Until Keyword Succeeds    15s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    IF    '${ODL_OF_PLUGIN}' == 'helium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=2    "output-node-connector":"2"=1
+    END
+    IF    '${ODL_OF_PLUGIN}' == 'lithium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=1    "output-node-connector":"2"=1
+    END
+    ClusterManagement.Put_As_Json_And_Check_Member_List_Or_All
+    ...    ${config_table_0}/flow=1
+    ...    ${body}
+    ...    ${controller_index}
+    ...    ${controller_index_list}
+    Wait Until Keyword Succeeds
+    ...    15s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_table_0}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Delete Sample Flow And Verify
-    [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Delete sample flow in Owner and verify it gets removed from all instances.
-    ${dictionary}=    Create Dictionary    10.0.2.0/24=0
-    Delete And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    [Arguments]    ${controller_index}    ${controller_index_list}=${EMPTY}
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=0
+    ClusterManagement.Delete_And_Check_Member_List_Or_All
+    ...    ${config_table_0}/flow=1
+    ...    ${controller_index}
+    ...    ${controller_index_list}
+    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_table_0}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Send RPC Add Sample Flow And Verify
-    [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Add sample flow in ${controller_index} and verify it gets applied from all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index}    ${controller_index_list}=${EMPTY}
     ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/add_flow_rpc.json
-    Run Keyword If    '${ODL_OF_PLUGIN}' == 'helium'    Set Test Variable    &{dictionary}    10.0.1.0/24=2
-    Run Keyword If    '${ODL_OF_PLUGIN}' == 'lithium'    Set Test Variable    &{dictionary}    10.0.1.0/24=1
-    ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:add-flow    data=${body}    headers=${HEADERS_YANG_JSON}
-    Log    ${resp.content}
-    Should Be Equal As Strings    ${resp.status_code}    200
-    Wait Until Keyword Succeeds    15s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    IF    '${ODL_OF_PLUGIN}' == 'helium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=2
+    END
+    IF    '${ODL_OF_PLUGIN}' == 'lithium'
+        Set Test Variable    &{dictionary}    10.0.1.0/24=1
+    END
+    ClusterManagement.Post_As_Json_To_Member
+    ...    uri=/rests/operations/sal-flow:add-flow
+    ...    data=${body}
+    ...    member_index=${controller_index}
+    Wait Until Keyword Succeeds
+    ...    15s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_table_0}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Send RPC Delete Sample Flow And Verify
-    [Arguments]    ${controller_index_list}    ${controller_index}
     [Documentation]    Delete sample flow in ${controller_index} and verify it gets removed from all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index}    ${controller_index_list}=${EMPTY}
     ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/delete_flow_rpc.json
     ${dictionary}=    Create Dictionary    10.0.1.0/24=0
-    ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:remove-flow    data=${body}    headers=${HEADERS_YANG_JSON}
-    Log    ${resp.content}
-    Should Be Equal As Strings    ${resp.status_code}    200
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+    ClusterManagement.Post_As_Json_To_Member
+    ...    uri=/rests/operations/sal-flow:remove-flow
+    ...    data=${body}
+    ...    member_index=${controller_index}
+    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_table_0}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Take OpenFlow Device Link Down and Verify
-    [Arguments]    ${controller_index_list}
     [Documentation]    Take a link down and verify port status in all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index_list}=${EMPTY}
     ${dictionary}=    Create Dictionary    "link-down":true=1
-    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 down
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+    ${ouput}=    MininetKeywords.Send Mininet Command    ${mininet_conn_id}    link s1 s2 down
+    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_port_1}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
+    ${dictionary}=    Create Dictionary    openflow:1=16    openflow:2=14    openflow:3=19
+    Wait Until Keyword Succeeds
+    ...    20s
+    ...    2s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_TOPO_API}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
 
 Take OpenFlow Device Link Up and Verify
-    [Arguments]    ${controller_index_list}
     [Documentation]    Take the link up and verify port status in all instances in ${controller_index_list}.
+    [Arguments]    ${controller_index_list}=${EMPTY}
     ${dictionary}=    Create Dictionary    "link-down":true=0
-    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 up
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+    ${ouput}=    MininetKeywords.Send Mininet Command    ${mininet_conn_id}    link s1 s2 up
+    Wait Until Keyword Succeeds
+    ...    5s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${operational_port_1}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
+    ${dictionary}=    Create Dictionary    openflow:1=21    openflow:2=19    openflow:3=19
+    Wait Until Keyword Succeeds
+    ...    10s
+    ...    1s
+    ...    ClusterManagement.Check_Item_Occurrence_Member_List_Or_All
+    ...    uri=${RFC8040_OPERATIONAL_TOPO_API}
+    ...    dictionary=${dictionary}
+    ...    member_index_list=${controller_index_list}
+
+Verify Switch Connections Running On Member
+    [Documentation]    Check if number of Switch connections on member of given index is equal to ${switch_count}.
+    [Arguments]    ${switch_count}    ${member_index}
+    ${count}=    ScaleClient.Get_Switches_Count    controller=${ODL_SYSTEM_${member_index}_IP}
+    BuiltIn.Should_Be_Equal_As_Numbers    ${switch_count}    ${count}
+
+Check Flows Operational Datastore On Member
+    [Documentation]    Check if number of Operational Flows on member of given index is equal to ${flow_count}.
+    [Arguments]    ${flow_count}    ${member_index}
+    ${sw}    ${reported_flow}    ${found_flow}=    ScaleClient.Flow Stats Collected
+    ...    controller=${ODL_SYSTEM_${member_index}_IP}
+    BuiltIn.Should_Be_Equal_As_Numbers    ${flow_count}    ${found_flow}
+
+Check Linear Topology On Member
+    [Documentation]    Check Linear topology.
+    [Arguments]    ${switches}    ${member_index}=1
+    ${session}=    Resolve_Http_Session_For_Member    member_index=${member_index}
+    ${resp}=    RequestsLibrary.Get Request    ${session}    ${RFC8040_OPERATIONAL_TOPO_API}
+    Log    ${resp.text}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    FOR    ${switch}    IN RANGE    1    ${switches+1}
+        Should Contain    ${resp.text}    "node-id":"openflow:${switch}"
+        Should Contain    ${resp.text}    "tp-id":"openflow:${switch}:1"
+        Should Contain    ${resp.text}    "tp-id":"openflow:${switch}:2"
+        Should Contain    ${resp.text}    "source-tp":"openflow:${switch}:2"
+        Should Contain    ${resp.text}    "dest-tp":"openflow:${switch}:2"
+        ${edge}=    Evaluate    ${switch}==1 or ${switch}==${switches}
+        IF    not ${edge}
+            Should Contain    ${resp.text}    "tp-id":"openflow:${switch}:3"
+        END
+        IF    not ${edge}
+            Should Contain    ${resp.text}    "source-tp":"openflow:${switch}:3"
+        END
+        IF    not ${edge}
+            Should Contain    ${resp.text}    "dest-tp":"openflow:${switch}:3
+        END
+    END
+
+Check No Switches On Member
+    [Documentation]    Check no switch is in topology
+    [Arguments]    ${switches}    ${member_index}=1
+    ${session}=    Resolve_Http_Session_For_Member    member_index=${member_index}
+    ${resp}=    RequestsLibrary.Get Request    ${session}    ${RFC8040_OPERATIONAL_TOPO_API}
+    Log    ${resp.text}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    FOR    ${switch}    IN RANGE    1    ${switches+1}
+        Should Not Contain    ${resp.text}    openflow:${switch}
+    END
+
+Check Number Of Flows On Member
+    [Documentation]    Check number of flows in the inventory.
+    [Arguments]    ${flows}    ${member_index}=1
+    ${session}=    Resolve_Http_Session_For_Member    member_index=${member_index}
+    ${resp}=    RequestsLibrary.Get Request    ${session}    ${RFC8040_OPERATIONAL_NODES_API}
+    Log    ${resp.text}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    ${count}=    Get Count    ${resp.text}    "priority"
+    Should Be Equal As Integers    ${count}    ${flows}
+
+Check Number Of Groups On Member
+    [Documentation]    Check number of groups in the inventory.
+    [Arguments]    ${groups}    ${member_index}=1
+    ${session}=    Resolve_Http_Session_For_Member    member_index=${member_index}
+    ${resp}=    RequestsLibrary.Get Request    ${session}    ${RFC8040_OPERATIONAL_NODES_API}
+    Log    ${resp.text}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    ${group_count}=    Get Count    ${resp.text}    "group-type"
+    Should Be Equal As Integers    ${group_count}    ${groups}