Patch for basic HA cluster test (2/2)

Add owner fail and recover test cases.
New Cluster OF resource to avoid code replication.
New cluster keywords to kill and start cluster instances.
Some repairs to ClusterKeyword library

Change-Id: Iaee102a6032c27eaaecb312e7ef37cf66c4ab836
Signed-off-by: Luis Gomez <ecelgp@gmail.com>

diff --git a/csit/libraries/ClusterKeywords.robot b/csit/libraries/ClusterKeywords.robot
index ccd29bec4a20e864a1afbd980738bcd5ae0b4498..a77228a0215ac49583d0ca5b8eac1fd219c958ba 100644 (file)
--- a/csit/libraries/ClusterKeywords.robot
+++ b/csit/libraries/ClusterKeywords.robot
@@ -6,7 +6,8 @@ Library           ClusterStateLibrary.py
 Resource          Utils.robot
 
 *** Variables ***
 Resource          Utils.robot
 
 *** Variables ***

-${smc_node}       /org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
+${jolokia_conf}    /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore
+${jolokia_oper}    /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore

 ${jolokia_read}    /jolokia/read/org.opendaylight.controller
 
 *** Keywords ***
 ${jolokia_read}    /jolokia/read/org.opendaylight.controller
 
 *** Keywords ***


@@ -69,12 +70,13 @@ Get Cluster Entity Owner Status
     Should Not Be Empty    ${entity_owner}    No owner found for ${device}
     ${owner}=    Replace String    ${entity_owner}    member-    ${EMPTY}
     ${owner}=    Convert To Integer    ${owner}
     Should Not Be Empty    ${entity_owner}    No owner found for ${device}
     ${owner}=    Replace String    ${entity_owner}    member-    ${EMPTY}
     ${owner}=    Convert To Integer    ${owner}

+    List Should Contain Value    ${controller_index_list}    ${owner}    Owner ${owner} not exisiting in ${controller_index_list}

     ${entity_candidates_list}=    Get From Dictionary    @{entity_list}[${entity_index}]    candidate
     ${list_length}=    Get Length    ${entity_candidates_list}
     ${entity_candidates_list}=    Get From Dictionary    @{entity_list}[${entity_index}]    candidate
     ${list_length}=    Get Length    ${entity_candidates_list}

-    Should Be Equal    ${list_length}    ${length}    Not enough or too many candidates for ${device}

     : FOR    ${entity_candidate}    IN    @{entity_candidates_list}
     \    ${candidate}=    Replace String    &{entity_candidate}[name]    member-    ${EMPTY}
     \    ${candidate}=    Convert To Integer    ${candidate}
     : FOR    ${entity_candidate}    IN    @{entity_candidates_list}
     \    ${candidate}=    Replace String    &{entity_candidate}[name]    member-    ${EMPTY}
     \    ${candidate}=    Convert To Integer    ${candidate}

+    \    List Should Contain Value    ${controller_index_list}    ${candidate}    Candidate ${candidate} not exisiting in ${controller_index_list}

     \    Run Keyword If    '${candidate}' != '${owner}'    Append To List    ${candidates_list}    ${candidate}
     [Return]    ${owner}    ${candidates_list}
 
     \    Run Keyword If    '${candidate}' != '${owner}'    Append To List    ${candidates_list}    ${candidate}
     [Return]    ${owner}    ${candidates_list}
 


@@ -111,6 +113,20 @@ Delete And Check At URI In Cluster
     \    Wait Until Keyword Succeeds    5s    1s    No Content From URI    controller${i}    ${uri}
     \    ...    ${headers}
 
     \    Wait Until Keyword Succeeds    5s    1s    No Content From URI    controller${i}    ${uri}
     \    ...    ${headers}
 

+Kill Multiple Controllers
+    [Arguments]    @{controller_index_list}
+    [Documentation]    Give this keyword a scalar or list of controllers to be stopped.
+    : FOR    ${i}    IN    @{controller_index_list}
+    \    ${output}=    Run Command On Controller    ${ODL_SYSTEM_${i}_IP}    ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh
+    \    Controller Down Check    ${ODL_SYSTEM_${i}_IP}
+
+Start Multiple Controllers
+    [Arguments]    ${timeout}    @{controller_index_list}
+    [Documentation]    Give this keyword a scalar or list of controllers to be started.
+    : FOR    ${i}    IN    @{controller_index_list}
+    \    ${output}=    Run Command On Controller    ${ODL_SYSTEM_${i}_IP}    ${WORKSPACE}/${BUNDLEFOLDER}/bin/start
+    \    Wait For Controller Sync    ${timeout}    ${ODL_SYSTEM_${i}_IP}
+

 Get Controller List
     [Arguments]    ${exclude_controller}=${EMPTY}
     [Documentation]    Creates a list of all controllers minus any excluded controller.
 Get Controller List
     [Arguments]    ${exclude_controller}=${EMPTY}
     [Documentation]    Creates a list of all controllers minus any excluded controller.


@@ -179,7 +195,7 @@ Controller Down Check
     [Arguments]    ${ip}
     [Documentation]    Checks to see if a controller is down by verifying that the karaf process isn't present.
     ${cmd} =    Set Variable    ps axf | grep karaf | grep -v grep | wc -l
     [Arguments]    ${ip}
     [Documentation]    Checks to see if a controller is down by verifying that the karaf process isn't present.
     ${cmd} =    Set Variable    ps axf | grep karaf | grep -v grep | wc -l

-    ${response}    Run Command On Remote System    ${ip}    ${cmd}
+    ${response}    Run Command On COntroller    ${ip}    ${cmd}

     Log    Number of controller instances running: ${response}
     Should Start With    ${response}    0    Controller process found or there may be extra instances of karaf running on the host machine.
 
     Log    Number of controller instances running: ${response}
     Should Start With    ${response}    0    Controller process found or there may be extra instances of karaf running on the host machine.
 


@@ -206,27 +222,32 @@ Controller Sync Status Should Be True
     [Arguments]    ${ip}
     [Documentation]    Checks if Sync Status is true.
     ${SyncStatus}=    Get Controller Sync Status    ${ip}
     [Arguments]    ${ip}
     [Documentation]    Checks if Sync Status is true.
     ${SyncStatus}=    Get Controller Sync Status    ${ip}

-    Should Be Equal    ${SyncStatus}    ${True}
+    Should Be Equal    ${SyncStatus}    ${TRUE}

 
 Controller Sync Status Should Be False
     [Arguments]    ${ip}
     [Documentation]    Checks if Sync Status is false.
     ${SyncStatus}=    Get Controller Sync Status    ${ip}
 
 Controller Sync Status Should Be False
     [Arguments]    ${ip}
     [Documentation]    Checks if Sync Status is false.
     ${SyncStatus}=    Get Controller Sync Status    ${ip}

-    Should Be Equal    ${SyncStatus}    ${False}
+    Should Be Equal    ${SyncStatus}    ${FALSE}

 
 Get Controller Sync Status
     [Arguments]    ${controller_ip}
     [Documentation]    Return Sync Status.
 
 Get Controller Sync Status
     [Arguments]    ${controller_ip}
     [Documentation]    Return Sync Status.

-    ${api}    Set Variable    /jolokia/read
-    Create_Session    session    http://${controller_ip}:${RESTCONFPORT}${api}    headers=${HEADERS}    auth=${AUTH}
-    ${resp}=    RequestsLibrary.Get    session    ${smc_node}
-    Log    ${resp.json()}
-    Log    ${resp.content}
-    ${json}=    Set Variable    ${resp.json()}
+    Create_Session    session    http://${controller_ip}:${RESTCONFPORT}    headers=${HEADERS}    auth=${AUTH}
+    ${data}=    Get Data From URI    session    ${jolokia_conf}
+    Log    ${data}
+    ${json}=    To Json    ${data}
+    ${value}=    Get From Dictionary    ${json}    value
+    ${ConfSyncStatus}=    Get From Dictionary    ${value}    SyncStatus
+    Log    Configuration Sync Status: ${ConfSyncStatus}
+    ${data}=    Get Data From URI    session    ${jolokia_oper}
+    Log    ${data}
+    ${json}=    To Json    ${data}

     ${value}=    Get From Dictionary    ${json}    value
     ${value}=    Get From Dictionary    ${json}    value

-    Log    value: ${value}
-    ${SyncStatus}=    Get From Dictionary    ${value}    SyncStatus
-    Log    SyncSatus: ${SyncStatus}
+    ${OperSyncStatus}=    Get From Dictionary    ${value}    SyncStatus
+    Log    Operational Sync Status: ${OperSyncStatus}
+    Run Keyword If    ${OperSyncStatus} and ${ConfSyncStatus}    Set Test Variable    ${SyncStatus}    ${TRUE}
+    ...    ELSE    Set Test Variable    ${SyncStatus}    ${FALSE}

     [Return]    ${SyncStatus}
 
 Clean One Or More Journals
     [Return]    ${SyncStatus}
 
 Clean One Or More Journals

diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot
new file mode 100644 (file)
index 0000000..3ecee4d
--- /dev/null
+++ b/csit/libraries/ClusterOpenFlow.robot
@@ -0,0 +1,107 @@
+*** Settings ***
+Documentation     Cluster OpenFlow library. So far this library is only to be used by OpenFlow cluster test as it is very specific for this test.
+Library           RequestsLibrary
+Resource          ClusterKeywords.robot
+Resource          MininetKeywords.robot
+Resource          Utils.robot
+Variables         ../variables/Variables.py
+
+*** Variables ***
+${config_table_0}    ${CONFIG_NODES_API}/node/openflow:1/table/0
+${operational_table_0}    ${OPERATIONAL_NODES_API}/node/openflow:1/table/0
+${operational_port_1}    ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector/openflow:1:1
+
+*** Keywords ***
+Check OpenFlow Shards Status
+    [Arguments]    ${controller_index_list}
+    [Documentation]    Check Status for all shards in OpenFlow application.
+    ${inv_conf_leader}    ${inv_conf_followers_list}    Wait Until Keyword Succeeds    10s    1s    Get Cluster Shard Status    ${controller_index_list}
+    ...    config    inventory
+    ${inv_oper_leader}    ${inv_oper_followers_list}    Wait Until Keyword Succeeds    10s    1s    Get Cluster Shard Status    ${controller_index_list}
+    ...    operational    inventory
+    ${topo_oper_leader}    ${topo_oper_followers_list}    Wait Until Keyword Succeeds    10s    1s    Get Cluster Shard Status    ${controller_index_list}
+    ...    operational    topology
+    Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
+    Log    operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list}
+    Log    operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list}
+
+Get OpenFlow Entity Owner Status For One Device
+    [Arguments]    ${controller_index_list}
+    [Documentation]    Check Entity Owner Status and identify owner and candidate.
+    ${owner}    ${candidates_list}    Wait Until Keyword Succeeds    10s    1s    Get Cluster Entity Owner Status    ${controller_index_list}
+    ...    openflow    openflow:1
+    [Return]    ${owner}    ${candidates_list}
+
+Check OpenFlow Network Operational Information For One Device
+    [Arguments]    ${controller_index_list}
+    [Documentation]    Check device openflow:1 is in operational inventory and topology in all instances in ${controller_index_list}.
+    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector.
+    ${dictionary}    Create Dictionary    openflow:1=4
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
+    ${dictionary}    Create Dictionary    openflow:1=11
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+
+Check No OpenFlow Network Operational Information
+    [Arguments]    ${controller_index_list}
+    [Documentation]    Check device is not in operational inventory or topology in all cluster instances in ${controller_index_list}.
+    ${dictionary}    Create Dictionary    openflow=0
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
+    ${dictionary}    Create Dictionary    openflow=0
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+
+Add Sample Flow And Verify
+    [Arguments]    ${controller_index_list}    ${controller_index}
+    [Documentation]    Add sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/sample_flow_1.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
+    Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+
+Modify Sample Flow And Verify
+    [Arguments]    ${controller_index_list}    ${controller_index}
+    [Documentation]    Modify sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/sample_flow_2.json
+    ${dictionary}=    Create Dictionary    10.0.2.0/24=1
+    Put And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+
+Delete Sample Flow And Verify
+    [Arguments]    ${controller_index_list}    ${controller_index}
+    [Documentation]    Delete sample flow in Owner and verify it gets removed from all instances.
+    ${dictionary}=    Create Dictionary    10.0.2.0/24=0
+    Delete And Check At URI In Cluster    ${controller_index_list}    ${controller_index}    ${config_table_0}/flow/1
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+
+Send RPC Add Sample Flow And Verify
+    [Arguments]    ${controller_index_list}    ${controller_index}
+    [Documentation]    Add sample flow in ${controller_index} and verify it gets applied from all instances in ${controller_index_list}.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/add_flow_rpc.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
+    ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:add-flow    ${body}    ${HEADERS}
+    Log    ${resp.content}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+
+Send RPC Delete Sample Flow And Verify
+    [Arguments]    ${controller_index_list}    ${controller_index}
+    [Documentation]    Delete sample flow in ${controller_index} and verify it gets removed from all instances in ${controller_index_list}.
+    ${body}=    OperatingSystem.Get File    ${CURDIR}/../variables/openflowplugin/delete_flow_rpc.json
+    ${dictionary}=    Create Dictionary    10.0.1.0/24=0
+    ${resp}    RequestsLibrary.Post Request    controller${controller_index}    /restconf/operations/sal-flow:remove-flow    ${body}    ${HEADERS}
+    Log    ${resp.content}
+    Should Be Equal As Strings    ${resp.status_code}    200
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_table_0}
+
+Take OpenFlow Device Link Down and Verify
+    [Arguments]    ${controller_index_list}
+    [Documentation]    Take a link down and verify port status in all instances in ${controller_index_list}.
+    ${dictionary}=    Create Dictionary    "link-down":true=1
+    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 down
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}
+
+Take OpenFlow Device Link Up and Verify
+    [Arguments]    ${controller_index_list}
+    [Documentation]    Take the link up and verify port status in all instances in ${controller_index_list}.
+    ${dictionary}=    Create Dictionary    "link-down":true=0
+    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 up
+    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${controller_index_list}    ${dictionary}    ${operational_port_1}

diff --git a/csit/libraries/KarafKeywords.robot b/csit/libraries/KarafKeywords.robot
index 200d2d2b114e17f66f7305c02358582c2ae35dd0..87f2f05818fda412a6481cd3f3d9caf573c1217d 100644 (file)
--- a/csit/libraries/KarafKeywords.robot
+++ b/csit/libraries/KarafKeywords.robot
@@ -1,4 +1,5 @@
 *** Settings ***
 *** Settings ***

+Documentation     Karaf library. This library is useful to deal with controller Karaf console.

 Library           SSHLibrary
 Library           OperatingSystem
 Variables         ../variables/Variables.py
 Library           SSHLibrary
 Library           OperatingSystem
 Variables         ../variables/Variables.py

diff --git a/csit/libraries/MininetKeywords.robot b/csit/libraries/MininetKeywords.robot
index f4c155ae3f7239f0867d0664ebe7af53335558c5..38cc17fff03b13fa252d835ef25ef7c0a1228fa7 100644 (file)
--- a/csit/libraries/MininetKeywords.robot
+++ b/csit/libraries/MininetKeywords.robot
@@ -1,4 +1,5 @@
 *** Settings ***
 *** Settings ***

+Documentation     Mininet library. This library is useful for tests using mininet tool to simulate devices.

 Library           SSHLibrary
 Resource          Utils.robot
 Variables         ../variables/Variables.py
 Library           SSHLibrary
 Resource          Utils.robot
 Variables         ../variables/Variables.py

diff --git a/csit/libraries/Utils.robot b/csit/libraries/Utils.robot
index df967dda00d78eed4e3bb0f2a9d9b2dcf695248f..85412c1aa28a04036d5af37d958f8aae757f9065 100644 (file)
--- a/csit/libraries/Utils.robot
+++ b/csit/libraries/Utils.robot
@@ -1,4 +1,5 @@
 *** Settings ***
 *** Settings ***

+Documentation     General Utils library. This library has broad scope, it can be used by any robot system tests.

 Library           SSHLibrary
 Library           String
 Library           DateTime
 Library           SSHLibrary
 Library           String
 Library           DateTime

diff --git a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
index ea6198ea312e88b0832856095e31b957323488bd..ff9e3c970fd00fe7e2fd89b23f75db80dd2c4b18 100644 (file)
--- a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
+++ b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot
@@ -1,136 +1,221 @@
 *** Settings ***
 *** Settings ***

+Documentation     Test suite for Cluster HA - Device Owner failover

 Suite Setup       Create Controller Sessions
 Suite Teardown    Delete All Sessions
 Library           RequestsLibrary
 Suite Setup       Create Controller Sessions
 Suite Teardown    Delete All Sessions
 Library           RequestsLibrary

+Resource          ../../../libraries/ClusterOpenFlow.robot

 Resource          ../../../libraries/ClusterKeywords.robot
 Resource          ../../../libraries/MininetKeywords.robot
 Variables         ../../../variables/Variables.py
 
 Resource          ../../../libraries/ClusterKeywords.robot
 Resource          ../../../libraries/MininetKeywords.robot
 Variables         ../../../variables/Variables.py
 

-*** Variables ***
-${config_table_0}    ${CONFIG_NODES_API}/node/openflow:1/table/0
-${operational_table_0}    ${OPERATIONAL_NODES_API}/node/openflow:1/table/0
-${operational_port_1}    ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector/openflow:1:1
-

 *** Test Cases ***
 *** Test Cases ***

-Check OpenFlow Shards Status
-    [Documentation]    Create original cluster list and Check Status for all shards in OpenFlow application.
+Create Original Cluster List
+    [Documentation]    Create original cluster list.

     ${original_cluster_list}    Create Controller Index List
     Set Suite Variable    ${original_cluster_list}
     ${original_cluster_list}    Create Controller Index List
     Set Suite Variable    ${original_cluster_list}

-    ${inv_conf_leader}    ${inv_conf_followers_list}    Get Cluster Shard Status    ${original_cluster_list}    config    inventory
-    ${inv_oper_leader}    ${inv_oper_followers_list}    Get Cluster Shard Status    ${original_cluster_list}    operational    inventory
-    ${topo_oper_leader}    ${topo_oper_followers_list}    Get Cluster Shard Status    ${original_cluster_list}    operational    topology
-    Log    config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list}
-    Log    operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list}
-    Log    operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list}
+
+Check Shards Status Before Fail
+    [Documentation]    Check Status for all shards in OpenFlow application.
+    Check OpenFlow Shards Status    ${original_cluster_list}

 
 Start Mininet Multiple Connections
     [Documentation]    Start mininet with connection to all cluster instances.
     ${mininet_conn_id}=    Start Mininet Multiple Controllers    ${TOOLS_SYSTEM_IP}    ${original_cluster_list}
     Set Suite Variable    ${mininet_conn_id}
 
 
 Start Mininet Multiple Connections
     [Documentation]    Start mininet with connection to all cluster instances.
     ${mininet_conn_id}=    Start Mininet Multiple Controllers    ${TOOLS_SYSTEM_IP}    ${original_cluster_list}
     Set Suite Variable    ${mininet_conn_id}
 

-Check Entity Owner Status And Find Owner and Candidate
+Check Entity Owner Status And Find Owner and Candidate Before Fail

     [Documentation]    Check Entity Owner Status and identify owner and candidate.
     [Documentation]    Check Entity Owner Status and identify owner and candidate.

-    ${original_owner}    ${original_candidates_list}    Wait Until Keyword Succeeds    5s    1s    Get Cluster Entity Owner Status    ${original_cluster_list}
-    ...    openflow    openflow:1
+    ${original_owner}    ${original_candidates_list}    Get OpenFlow Entity Owner Status For One Device    ${original_cluster_list}

     ${original_candidate}=    Get From List    ${original_candidates_list}    0
     Set Suite Variable    ${original_owner}
     Set Suite Variable    ${original_candidate}
 
     ${original_candidate}=    Get From List    ${original_candidates_list}    0
     Set Suite Variable    ${original_owner}
     Set Suite Variable    ${original_candidate}
 

-Check Network Operational Information
+Check Network Operational Information Before Fail

     [Documentation]    Check device is in operational inventory and topology in all cluster instances.
     [Documentation]    Check device is in operational inventory and topology in all cluster instances.

-    ...    Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector.
-    ${dictionary}    Create Dictionary    openflow:1=4
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
-    ${dictionary}    Create Dictionary    openflow:1=11
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+    Check OpenFlow Network Operational Information For One Device    ${original_cluster_list}

 
 

-Add Flow In Owner and Verify
+Add Configuration In Owner and Verify Before Fail

     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_1.json
-    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
-    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_owner}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+    Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 

-Modify Flow In Owner and Verify
+Modify Configuration In Owner and Verify Before Fail

     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_2.json
-    ${dictionary}=    Create Dictionary    10.0.2.0/24=1
-    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_owner}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+    Modify Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 

-Delete Flow In Owner and Verify
+Delete Configuration In Owner and Verify Before Fail

     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
     [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.

-    ${dictionary}=    Create Dictionary    10.0.2.0/24=0
-    Delete And Check At URI In Cluster    ${original_cluster_list}    ${original_owner}    ${config_table_0}/flow/1
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+    Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}

 
 

-Add Flow In Candidate and Verify
+Add Configuration In Candidate and Verify Before Fail

     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_1.json
-    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
-    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_candidate}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+    Add Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 

-Modify Flow In Candidate and Verify
+Modify Configuration In Candidate and Verify Before Fail

     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
     [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/sample_flow_2.json
-    ${dictionary}=    Create Dictionary    10.0.2.0/24=1
-    Put And Check At URI In Cluster    ${original_cluster_list}    ${original_candidate}    ${config_table_0}/flow/1    ${body}    ${HEADERS}
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+    Modify Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 

-Delete Flow In Candidate and Verify
+Delete Configuration In Candidate and Verify Before Fail

     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    ${dictionary}=    Create Dictionary    10.0.2.0/24=0
-    Delete And Check At URI In Cluster    ${original_cluster_list}    ${original_candidate}    ${config_table_0}/flow/1
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
+    Delete Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}

 
 

-Send RPC Add Flow to Owner and Verify
+Send RPC Add to Owner and Verify Before Fail

     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
     [Documentation]    Add Flow in Owner and verify it gets applied from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/add_flow_rpc.json
-    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
-    ${resp}    RequestsLibrary.Post Request    controller${original_owner}    /restconf/operations/sal-flow:add-flow    ${body}    ${HEADERS}
-    Log    ${resp.content}
-    Should Be Equal As Strings    ${resp.status_code}    200
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
-
-Send RPC Delete Flow to Owner and Verify
+    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+
+Send RPC Delete to Owner and Verify Before Fail

     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
     [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/delete_flow_rpc.json
-    ${dictionary}=    Create Dictionary    10.0.1.0/24=0
-    ${resp}    RequestsLibrary.Post Request    controller${original_owner}    /restconf/operations/sal-flow:remove-flow    ${body}    ${HEADERS}
-    Log    ${resp.content}
-    Should Be Equal As Strings    ${resp.status_code}    200
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
-
-Send RPC Add Flow to Candidate and Verify
+    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+
+Send RPC Add to Candidate and Verify Before Fail

     [Documentation]    Add Flow in Candidate and verify it gets applied from all instances.
     [Documentation]    Add Flow in Candidate and verify it gets applied from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/add_flow_rpc.json
-    ${dictionary}=    Create Dictionary    10.0.1.0/24=1
-    ${resp}    RequestsLibrary.Post Request    controller${original_candidate}    /restconf/operations/sal-flow:add-flow    ${body}    ${HEADERS}
-    Log    ${resp.content}
-    Should Be Equal As Strings    ${resp.status_code}    200
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
-
-Send RPC Delete Flow to Candidate and Verify
+    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+
+Send RPC Delete to Candidate and Verify Before Fail

     [Documentation]    Delete Flow in Candidate and verify it gets removed from all instances.
     [Documentation]    Delete Flow in Candidate and verify it gets removed from all instances.

-    ${body}=    OperatingSystem.Get File    ${CURDIR}/../../../variables/openflowplugin/delete_flow_rpc.json
-    ${dictionary}=    Create Dictionary    10.0.1.0/24=0
-    ${resp}    RequestsLibrary.Post Request    controller${original_candidate}    /restconf/operations/sal-flow:remove-flow    ${body}    ${HEADERS}
-    Log    ${resp.content}
-    Should Be Equal As Strings    ${resp.status_code}    200
-    Wait Until Keyword Succeeds    10s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_table_0}
-
-Take a Link Down and Verify
+    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_candidate}
+
+Modify Network And Verify Before Fail
+    [Documentation]    Take a link down and verify port status in all instances.
+    Take OpenFlow Device Link Down and Verify    ${original_cluster_list}
+
+Restore Network And Verify Before Fail
+    [Documentation]    Take the link up and verify port status in all instances.
+    Take OpenFlow Device Link Up and Verify    ${original_cluster_list}
+
+Kill Owner Instance
+    [Documentation]    Kill Owner Instance and verify it is dead
+    Kill Multiple Controllers    ${original_owner}
+    ${new_cluster_list}    Create Controller Index List
+    Remove Values From List    ${new_cluster_list}    ${original_owner}
+    Set Suite Variable    ${new_cluster_list}
+
+Check Shards Status After Fail
+    [Documentation]    Create original cluster list and check Status for all shards in OpenFlow application.
+    Check OpenFlow Shards Status    ${new_cluster_list}
+
+Check Entity Owner Status And Find Owner and Candidate After Fail
+    [Documentation]    Check Entity Owner Status and identify owner and candidate.
+    ${new_owner}    ${new_candidates_list}    Get OpenFlow Entity Owner Status For One Device    ${new_cluster_list}
+    ${new_candidate}=    Get From List    ${new_candidates_list}    0
+    Set Suite Variable    ${new_owner}
+    Set Suite Variable    ${new_candidate}
+
+Check Network Operational Information After Fail
+    [Documentation]    Check device is in operational inventory and topology in all cluster instances.
+    Check OpenFlow Network Operational Information For One Device    ${new_cluster_list}
+
+Add Configuration In Owner and Verify After Fail
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Add Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+
+Modify Configuration In Owner and Verify After Fail
+    [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
+    Modify Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+
+Delete Configuration In Owner and Verify After Fail
+    [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
+    Delete Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+
+Add Configuration In Candidate and Verify After Fail
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Add Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+
+Modify Configuration In Candidate and Verify After Fail
+    [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
+    Modify Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+
+Delete Configuration In Candidate and Verify After Fail
+    [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
+    Delete Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+
+Send RPC Add to Owner and Verify After Fail
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Send RPC Add Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+
+Send RPC Delete to Owner and Verify After Fail
+    [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
+    Send RPC Delete Sample Flow and Verify    ${new_cluster_list}    ${new_owner}
+
+Send RPC Add to Candidate and Verify After Fail
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Send RPC Add Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+
+Send RPC Delete to Candidate and Verify After Fail
+    [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
+    Send RPC Delete Sample Flow and Verify    ${new_cluster_list}    ${new_candidate}
+
+Modify Network and Verify After Fail
+    [Documentation]    Take a link down and verify port status in all instances.
+    Take OpenFlow Device Link Down and Verify    ${new_cluster_list}
+
+Restore Network and Verify After Fail
+    [Documentation]    Take the link up and verify port status in all instances.
+    Take OpenFlow Device Link Up and Verify    ${new_cluster_list}
+
+Start Old Owner Instance
+    [Documentation]    Kill Owner Instance and verify it is dead
+    Start Multiple Controllers    300s    ${original_owner}
+
+Check Shards Status After Recover
+    [Documentation]    Create original cluster list and check Status for all shards in OpenFlow application.
+    Wait Until Keyword Succeeds    5s    1s    Check OpenFlow Shards Status    ${original_cluster_list}
+
+Check Entity Owner Status After Recover
+    [Documentation]    Check Entity Owner Status and identify owner and candidate.
+    ${new_owner}    ${new_candidates_list}    Wait Until Keyword Succeeds    5s    1s    Get OpenFlow Entity Owner Status For One Device    ${original_cluster_list}
+    Set Suite Variable    ${new_owner}
+
+Check Network Operational Information After Recover
+    [Documentation]    Check device is in operational inventory and topology in all cluster instances.
+    Check OpenFlow Network Operational Information For One Device    ${original_cluster_list}
+
+Add Configuration In Owner and Verify After Recover
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Add Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+
+Modify Configuration In Owner and Verify After Recover
+    [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
+    Modify Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+
+Delete Configuration In Owner and Verify After Recover
+    [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
+    Delete Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+
+Add Configuration In Old Owner and Verify After Recover
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Add Sample Flow and Verify    ${originalcluster_list}    ${original_owner}
+
+Modify Configuration In Old Owner and Verify After Recover
+    [Documentation]    Modify Flow in Owner and verify it gets applied from all instances.
+    Modify Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+
+Delete Configuration In Old Owner and Verify After Recover
+    [Documentation]    Delete Flow in Owner and verify it gets applied from all instances.
+    Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+
+Send RPC Add to Owner and Verify After Recover
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+
+Send RPC Delete to Owner and Verify After Recover
+    [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
+    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${new_owner}
+
+Send RPC Add to Old Owner and Verify After Recover
+    [Documentation]    Add Flow in Owner and verify it gets applied from all instances.
+    Send RPC Add Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+
+Send RPC Delete to Old Owner and Verify After Recover
+    [Documentation]    Delete Flow in Owner and verify it gets removed from all instances.
+    Send RPC Delete Sample Flow and Verify    ${original_cluster_list}    ${original_owner}
+
+Modify Network and Verify After Recover

     [Documentation]    Take a link down and verify port status in all instances.
     [Documentation]    Take a link down and verify port status in all instances.

-    ${dictionary}=    Create Dictionary    "link-down":true=1
-    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 down
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_port_1}
+    Take OpenFlow Device Link Down and Verify    ${original_cluster_list}

 
 

-Take a Link Up and Verify
+Restore Network and Verify After Recover

     [Documentation]    Take the link up and verify port status in all instances.
     [Documentation]    Take the link up and verify port status in all instances.

-    ${dictionary}=    Create Dictionary    "link-down":true=0
-    ${ouput}=    Send Mininet Command    ${mininet_conn_id}    link s1 h1 up
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${operational_port_1}
+    Take OpenFlow Device Link Up and Verify    ${original_cluster_list}

 
 Stop Mininet and Exit
     [Documentation]    Stop mininet and exit connection.
 
 Stop Mininet and Exit
     [Documentation]    Stop mininet and exit connection.


@@ -139,7 +224,4 @@ Stop Mininet and Exit
 
 Check No Network Operational Information
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances.
 
 Check No Network Operational Information
     [Documentation]    Check device is not in operational inventory or topology in all cluster instances.

-    ${dictionary}    Create Dictionary    openflow:1=0
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_NODES_API}
-    ${dictionary}    Create Dictionary    openflow:1=0
-    Wait Until Keyword Succeeds    5s    1s    Check Item Occurrence At URI In Cluster    ${original_cluster_list}    ${dictionary}    ${OPERATIONAL_TOPO_API}
+    Check No OpenFlow Network Operational Information    ${original_cluster_list}

diff --git a/csit/testplans/openflowplugin-clustering.txt b/csit/testplans/openflowplugin-clustering.txt
index b200f39fc90d88ee40ed174a75d6605ffc485557..42ec6a2e1f2d79d27ce79d419b59ebcea447b082 100644 (file)
--- a/csit/testplans/openflowplugin-clustering.txt
+++ b/csit/testplans/openflowplugin-clustering.txt
@@ -1,3 +1,4 @@
 # Place the suites in run order:
 integration/test/csit/suites/openflowplugin/Clustering
 # Place the suites in run order:
 integration/test/csit/suites/openflowplugin/Clustering

+integration/test/csit/suites/openflowplugin/Clustering

 integration/test/csit/suites/openflowplugin/Sanity3Node
 integration/test/csit/suites/openflowplugin/Sanity3Node