From 776255eefda71fa0233b9cc4efe8d63f7d7ea305 Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Thu, 12 Nov 2015 22:10:01 -0800 Subject: [PATCH] Patch for basic HA cluster test (2/2) Add owner fail and recover test cases. New Cluster OF resource to avoid code replication. New cluster keywords to kill and start cluster instances. Some repairs to ClusterKeyword library Change-Id: Iaee102a6032c27eaaecb312e7ef37cf66c4ab836 Signed-off-by: Luis Gomez --- csit/libraries/ClusterKeywords.robot | 49 +++- csit/libraries/ClusterOpenFlow.robot | 107 +++++++ csit/libraries/KarafKeywords.robot | 1 + csit/libraries/MininetKeywords.robot | 1 + csit/libraries/Utils.robot | 1 + .../010__Cluster_HA_Owner_Failover.robot | 270 ++++++++++++------ csit/testplans/openflowplugin-clustering.txt | 1 + 7 files changed, 322 insertions(+), 108 deletions(-) create mode 100644 csit/libraries/ClusterOpenFlow.robot diff --git a/csit/libraries/ClusterKeywords.robot b/csit/libraries/ClusterKeywords.robot index ccd29bec4a..a77228a021 100644 --- a/csit/libraries/ClusterKeywords.robot +++ b/csit/libraries/ClusterKeywords.robot @@ -6,7 +6,8 @@ Library ClusterStateLibrary.py Resource Utils.robot *** Variables *** -${smc_node} /org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore +${jolokia_conf} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-config,type=DistributedConfigDatastore +${jolokia_oper} /jolokia/read/org.opendaylight.controller:Category=ShardManager,name=shard-manager-operational,type=DistributedOperationalDatastore ${jolokia_read} /jolokia/read/org.opendaylight.controller *** Keywords *** @@ -69,12 +70,13 @@ Get Cluster Entity Owner Status Should Not Be Empty ${entity_owner} No owner found for ${device} ${owner}= Replace String ${entity_owner} member- ${EMPTY} ${owner}= Convert To Integer ${owner} + List Should Contain Value ${controller_index_list} ${owner} Owner ${owner} not exisiting in ${controller_index_list} ${entity_candidates_list}= Get From Dictionary @{entity_list}[${entity_index}] candidate ${list_length}= Get Length ${entity_candidates_list} - Should Be Equal ${list_length} ${length} Not enough or too many candidates for ${device} : FOR ${entity_candidate} IN @{entity_candidates_list} \ ${candidate}= Replace String &{entity_candidate}[name] member- ${EMPTY} \ ${candidate}= Convert To Integer ${candidate} + \ List Should Contain Value ${controller_index_list} ${candidate} Candidate ${candidate} not exisiting in ${controller_index_list} \ Run Keyword If '${candidate}' != '${owner}' Append To List ${candidates_list} ${candidate} [Return] ${owner} ${candidates_list} @@ -111,6 +113,20 @@ Delete And Check At URI In Cluster \ Wait Until Keyword Succeeds 5s 1s No Content From URI controller${i} ${uri} \ ... ${headers} +Kill Multiple Controllers + [Arguments] @{controller_index_list} + [Documentation] Give this keyword a scalar or list of controllers to be stopped. + : FOR ${i} IN @{controller_index_list} + \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh + \ Controller Down Check ${ODL_SYSTEM_${i}_IP} + +Start Multiple Controllers + [Arguments] ${timeout} @{controller_index_list} + [Documentation] Give this keyword a scalar or list of controllers to be started. + : FOR ${i} IN @{controller_index_list} + \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start + \ Wait For Controller Sync ${timeout} ${ODL_SYSTEM_${i}_IP} + Get Controller List [Arguments] ${exclude_controller}=${EMPTY} [Documentation] Creates a list of all controllers minus any excluded controller. @@ -179,7 +195,7 @@ Controller Down Check [Arguments] ${ip} [Documentation] Checks to see if a controller is down by verifying that the karaf process isn't present. ${cmd} = Set Variable ps axf | grep karaf | grep -v grep | wc -l - ${response} Run Command On Remote System ${ip} ${cmd} + ${response} Run Command On COntroller ${ip} ${cmd} Log Number of controller instances running: ${response} Should Start With ${response} 0 Controller process found or there may be extra instances of karaf running on the host machine. @@ -206,27 +222,32 @@ Controller Sync Status Should Be True [Arguments] ${ip} [Documentation] Checks if Sync Status is true. ${SyncStatus}= Get Controller Sync Status ${ip} - Should Be Equal ${SyncStatus} ${True} + Should Be Equal ${SyncStatus} ${TRUE} Controller Sync Status Should Be False [Arguments] ${ip} [Documentation] Checks if Sync Status is false. ${SyncStatus}= Get Controller Sync Status ${ip} - Should Be Equal ${SyncStatus} ${False} + Should Be Equal ${SyncStatus} ${FALSE} Get Controller Sync Status [Arguments] ${controller_ip} [Documentation] Return Sync Status. - ${api} Set Variable /jolokia/read - Create_Session session http://${controller_ip}:${RESTCONFPORT}${api} headers=${HEADERS} auth=${AUTH} - ${resp}= RequestsLibrary.Get session ${smc_node} - Log ${resp.json()} - Log ${resp.content} - ${json}= Set Variable ${resp.json()} + Create_Session session http://${controller_ip}:${RESTCONFPORT} headers=${HEADERS} auth=${AUTH} + ${data}= Get Data From URI session ${jolokia_conf} + Log ${data} + ${json}= To Json ${data} + ${value}= Get From Dictionary ${json} value + ${ConfSyncStatus}= Get From Dictionary ${value} SyncStatus + Log Configuration Sync Status: ${ConfSyncStatus} + ${data}= Get Data From URI session ${jolokia_oper} + Log ${data} + ${json}= To Json ${data} ${value}= Get From Dictionary ${json} value - Log value: ${value} - ${SyncStatus}= Get From Dictionary ${value} SyncStatus - Log SyncSatus: ${SyncStatus} + ${OperSyncStatus}= Get From Dictionary ${value} SyncStatus + Log Operational Sync Status: ${OperSyncStatus} + Run Keyword If ${OperSyncStatus} and ${ConfSyncStatus} Set Test Variable ${SyncStatus} ${TRUE} + ... ELSE Set Test Variable ${SyncStatus} ${FALSE} [Return] ${SyncStatus} Clean One Or More Journals diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot new file mode 100644 index 0000000000..3ecee4dc9f --- /dev/null +++ b/csit/libraries/ClusterOpenFlow.robot @@ -0,0 +1,107 @@ +*** Settings *** +Documentation Cluster OpenFlow library. So far this library is only to be used by OpenFlow cluster test as it is very specific for this test. +Library RequestsLibrary +Resource ClusterKeywords.robot +Resource MininetKeywords.robot +Resource Utils.robot +Variables ../variables/Variables.py + +*** Variables *** +${config_table_0} ${CONFIG_NODES_API}/node/openflow:1/table/0 +${operational_table_0} ${OPERATIONAL_NODES_API}/node/openflow:1/table/0 +${operational_port_1} ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector/openflow:1:1 + +*** Keywords *** +Check OpenFlow Shards Status + [Arguments] ${controller_index_list} + [Documentation] Check Status for all shards in OpenFlow application. + ${inv_conf_leader} ${inv_conf_followers_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Shard Status ${controller_index_list} + ... config inventory + ${inv_oper_leader} ${inv_oper_followers_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Shard Status ${controller_index_list} + ... operational inventory + ${topo_oper_leader} ${topo_oper_followers_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Shard Status ${controller_index_list} + ... operational topology + Log config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list} + Log operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list} + Log operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list} + +Get OpenFlow Entity Owner Status For One Device + [Arguments] ${controller_index_list} + [Documentation] Check Entity Owner Status and identify owner and candidate. + ${owner} ${candidates_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Entity Owner Status ${controller_index_list} + ... openflow openflow:1 + [Return] ${owner} ${candidates_list} + +Check OpenFlow Network Operational Information For One Device + [Arguments] ${controller_index_list} + [Documentation] Check device openflow:1 is in operational inventory and topology in all instances in ${controller_index_list}. + ... Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector. + ${dictionary} Create Dictionary openflow:1=4 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_NODES_API} + ${dictionary} Create Dictionary openflow:1=11 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} + +Check No OpenFlow Network Operational Information + [Arguments] ${controller_index_list} + [Documentation] Check device is not in operational inventory or topology in all cluster instances in ${controller_index_list}. + ${dictionary} Create Dictionary openflow=0 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_NODES_API} + ${dictionary} Create Dictionary openflow=0 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} + +Add Sample Flow And Verify + [Arguments] ${controller_index_list} ${controller_index} + [Documentation] Add sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/sample_flow_1.json + ${dictionary}= Create Dictionary 10.0.1.0/24=1 + Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} ${HEADERS} + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + +Modify Sample Flow And Verify + [Arguments] ${controller_index_list} ${controller_index} + [Documentation] Modify sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/sample_flow_2.json + ${dictionary}= Create Dictionary 10.0.2.0/24=1 + Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} ${HEADERS} + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + +Delete Sample Flow And Verify + [Arguments] ${controller_index_list} ${controller_index} + [Documentation] Delete sample flow in Owner and verify it gets removed from all instances. + ${dictionary}= Create Dictionary 10.0.2.0/24=0 + Delete And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + +Send RPC Add Sample Flow And Verify + [Arguments] ${controller_index_list} ${controller_index} + [Documentation] Add sample flow in ${controller_index} and verify it gets applied from all instances in ${controller_index_list}. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/add_flow_rpc.json + ${dictionary}= Create Dictionary 10.0.1.0/24=1 + ${resp} RequestsLibrary.Post Request controller${controller_index} /restconf/operations/sal-flow:add-flow ${body} ${HEADERS} + Log ${resp.content} + Should Be Equal As Strings ${resp.status_code} 200 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + +Send RPC Delete Sample Flow And Verify + [Arguments] ${controller_index_list} ${controller_index} + [Documentation] Delete sample flow in ${controller_index} and verify it gets removed from all instances in ${controller_index_list}. + ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/delete_flow_rpc.json + ${dictionary}= Create Dictionary 10.0.1.0/24=0 + ${resp} RequestsLibrary.Post Request controller${controller_index} /restconf/operations/sal-flow:remove-flow ${body} ${HEADERS} + Log ${resp.content} + Should Be Equal As Strings ${resp.status_code} 200 + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + +Take OpenFlow Device Link Down and Verify + [Arguments] ${controller_index_list} + [Documentation] Take a link down and verify port status in all instances in ${controller_index_list}. + ${dictionary}= Create Dictionary "link-down":true=1 + ${ouput}= Send Mininet Command ${mininet_conn_id} link s1 h1 down + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_port_1} + +Take OpenFlow Device Link Up and Verify + [Arguments] ${controller_index_list} + [Documentation] Take the link up and verify port status in all instances in ${controller_index_list}. + ${dictionary}= Create Dictionary "link-down":true=0 + ${ouput}= Send Mininet Command ${mininet_conn_id} link s1 h1 up + Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_port_1} diff --git a/csit/libraries/KarafKeywords.robot b/csit/libraries/KarafKeywords.robot index 200d2d2b11..87f2f05818 100644 --- a/csit/libraries/KarafKeywords.robot +++ b/csit/libraries/KarafKeywords.robot @@ -1,4 +1,5 @@ *** Settings *** +Documentation Karaf library. This library is useful to deal with controller Karaf console. Library SSHLibrary Library OperatingSystem Variables ../variables/Variables.py diff --git a/csit/libraries/MininetKeywords.robot b/csit/libraries/MininetKeywords.robot index f4c155ae3f..38cc17fff0 100644 --- a/csit/libraries/MininetKeywords.robot +++ b/csit/libraries/MininetKeywords.robot @@ -1,4 +1,5 @@ *** Settings *** +Documentation Mininet library. This library is useful for tests using mininet tool to simulate devices. Library SSHLibrary Resource Utils.robot Variables ../variables/Variables.py diff --git a/csit/libraries/Utils.robot b/csit/libraries/Utils.robot index df967dda00..85412c1aa2 100644 --- a/csit/libraries/Utils.robot +++ b/csit/libraries/Utils.robot @@ -1,4 +1,5 @@ *** Settings *** +Documentation General Utils library. This library has broad scope, it can be used by any robot system tests. Library SSHLibrary Library String Library DateTime diff --git a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot index ea6198ea31..ff9e3c970f 100644 --- a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot +++ b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot @@ -1,136 +1,221 @@ *** Settings *** +Documentation Test suite for Cluster HA - Device Owner failover Suite Setup Create Controller Sessions Suite Teardown Delete All Sessions Library RequestsLibrary +Resource ../../../libraries/ClusterOpenFlow.robot Resource ../../../libraries/ClusterKeywords.robot Resource ../../../libraries/MininetKeywords.robot Variables ../../../variables/Variables.py -*** Variables *** -${config_table_0} ${CONFIG_NODES_API}/node/openflow:1/table/0 -${operational_table_0} ${OPERATIONAL_NODES_API}/node/openflow:1/table/0 -${operational_port_1} ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector/openflow:1:1 - *** Test Cases *** -Check OpenFlow Shards Status - [Documentation] Create original cluster list and Check Status for all shards in OpenFlow application. +Create Original Cluster List + [Documentation] Create original cluster list. ${original_cluster_list} Create Controller Index List Set Suite Variable ${original_cluster_list} - ${inv_conf_leader} ${inv_conf_followers_list} Get Cluster Shard Status ${original_cluster_list} config inventory - ${inv_oper_leader} ${inv_oper_followers_list} Get Cluster Shard Status ${original_cluster_list} operational inventory - ${topo_oper_leader} ${topo_oper_followers_list} Get Cluster Shard Status ${original_cluster_list} operational topology - Log config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list} - Log operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list} - Log operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list} + +Check Shards Status Before Fail + [Documentation] Check Status for all shards in OpenFlow application. + Check OpenFlow Shards Status ${original_cluster_list} Start Mininet Multiple Connections [Documentation] Start mininet with connection to all cluster instances. ${mininet_conn_id}= Start Mininet Multiple Controllers ${TOOLS_SYSTEM_IP} ${original_cluster_list} Set Suite Variable ${mininet_conn_id} -Check Entity Owner Status And Find Owner and Candidate +Check Entity Owner Status And Find Owner and Candidate Before Fail [Documentation] Check Entity Owner Status and identify owner and candidate. - ${original_owner} ${original_candidates_list} Wait Until Keyword Succeeds 5s 1s Get Cluster Entity Owner Status ${original_cluster_list} - ... openflow openflow:1 + ${original_owner} ${original_candidates_list} Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} ${original_candidate}= Get From List ${original_candidates_list} 0 Set Suite Variable ${original_owner} Set Suite Variable ${original_candidate} -Check Network Operational Information +Check Network Operational Information Before Fail [Documentation] Check device is in operational inventory and topology in all cluster instances. - ... Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector. - ${dictionary} Create Dictionary openflow:1=4 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${OPERATIONAL_NODES_API} - ${dictionary} Create Dictionary openflow:1=11 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${OPERATIONAL_TOPO_API} + Check OpenFlow Network Operational Information For One Device ${original_cluster_list} -Add Flow In Owner and Verify +Add Configuration In Owner and Verify Before Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/sample_flow_1.json - ${dictionary}= Create Dictionary 10.0.1.0/24=1 - Put And Check At URI In Cluster ${original_cluster_list} ${original_owner} ${config_table_0}/flow/1 ${body} ${HEADERS} - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} + Add Sample Flow and Verify ${original_cluster_list} ${original_owner} -Modify Flow In Owner and Verify +Modify Configuration In Owner and Verify Before Fail [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/sample_flow_2.json - ${dictionary}= Create Dictionary 10.0.2.0/24=1 - Put And Check At URI In Cluster ${original_cluster_list} ${original_owner} ${config_table_0}/flow/1 ${body} ${HEADERS} - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} + Modify Sample Flow and Verify ${original_cluster_list} ${original_owner} -Delete Flow In Owner and Verify +Delete Configuration In Owner and Verify Before Fail [Documentation] Delete Flow in Owner and verify it gets applied from all instances. - ${dictionary}= Create Dictionary 10.0.2.0/24=0 - Delete And Check At URI In Cluster ${original_cluster_list} ${original_owner} ${config_table_0}/flow/1 - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} + Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} -Add Flow In Candidate and Verify +Add Configuration In Candidate and Verify Before Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/sample_flow_1.json - ${dictionary}= Create Dictionary 10.0.1.0/24=1 - Put And Check At URI In Cluster ${original_cluster_list} ${original_candidate} ${config_table_0}/flow/1 ${body} ${HEADERS} - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} + Add Sample Flow and Verify ${original_cluster_list} ${original_candidate} -Modify Flow In Candidate and Verify +Modify Configuration In Candidate and Verify Before Fail [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/sample_flow_2.json - ${dictionary}= Create Dictionary 10.0.2.0/24=1 - Put And Check At URI In Cluster ${original_cluster_list} ${original_candidate} ${config_table_0}/flow/1 ${body} ${HEADERS} - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} + Modify Sample Flow and Verify ${original_cluster_list} ${original_candidate} -Delete Flow In Candidate and Verify +Delete Configuration In Candidate and Verify Before Fail [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - ${dictionary}= Create Dictionary 10.0.2.0/24=0 - Delete And Check At URI In Cluster ${original_cluster_list} ${original_candidate} ${config_table_0}/flow/1 - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} + Delete Sample Flow and Verify ${original_cluster_list} ${original_candidate} -Send RPC Add Flow to Owner and Verify +Send RPC Add to Owner and Verify Before Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/add_flow_rpc.json - ${dictionary}= Create Dictionary 10.0.1.0/24=1 - ${resp} RequestsLibrary.Post Request controller${original_owner} /restconf/operations/sal-flow:add-flow ${body} ${HEADERS} - Log ${resp.content} - Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} - -Send RPC Delete Flow to Owner and Verify + Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_owner} + +Send RPC Delete to Owner and Verify Before Fail [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/delete_flow_rpc.json - ${dictionary}= Create Dictionary 10.0.1.0/24=0 - ${resp} RequestsLibrary.Post Request controller${original_owner} /restconf/operations/sal-flow:remove-flow ${body} ${HEADERS} - Log ${resp.content} - Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} - -Send RPC Add Flow to Candidate and Verify + Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + +Send RPC Add to Candidate and Verify Before Fail [Documentation] Add Flow in Candidate and verify it gets applied from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/add_flow_rpc.json - ${dictionary}= Create Dictionary 10.0.1.0/24=1 - ${resp} RequestsLibrary.Post Request controller${original_candidate} /restconf/operations/sal-flow:add-flow ${body} ${HEADERS} - Log ${resp.content} - Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} - -Send RPC Delete Flow to Candidate and Verify + Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_candidate} + +Send RPC Delete to Candidate and Verify Before Fail [Documentation] Delete Flow in Candidate and verify it gets removed from all instances. - ${body}= OperatingSystem.Get File ${CURDIR}/../../../variables/openflowplugin/delete_flow_rpc.json - ${dictionary}= Create Dictionary 10.0.1.0/24=0 - ${resp} RequestsLibrary.Post Request controller${original_candidate} /restconf/operations/sal-flow:remove-flow ${body} ${HEADERS} - Log ${resp.content} - Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 10s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_table_0} - -Take a Link Down and Verify + Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_candidate} + +Modify Network And Verify Before Fail + [Documentation] Take a link down and verify port status in all instances. + Take OpenFlow Device Link Down and Verify ${original_cluster_list} + +Restore Network And Verify Before Fail + [Documentation] Take the link up and verify port status in all instances. + Take OpenFlow Device Link Up and Verify ${original_cluster_list} + +Kill Owner Instance + [Documentation] Kill Owner Instance and verify it is dead + Kill Multiple Controllers ${original_owner} + ${new_cluster_list} Create Controller Index List + Remove Values From List ${new_cluster_list} ${original_owner} + Set Suite Variable ${new_cluster_list} + +Check Shards Status After Fail + [Documentation] Create original cluster list and check Status for all shards in OpenFlow application. + Check OpenFlow Shards Status ${new_cluster_list} + +Check Entity Owner Status And Find Owner and Candidate After Fail + [Documentation] Check Entity Owner Status and identify owner and candidate. + ${new_owner} ${new_candidates_list} Get OpenFlow Entity Owner Status For One Device ${new_cluster_list} + ${new_candidate}= Get From List ${new_candidates_list} 0 + Set Suite Variable ${new_owner} + Set Suite Variable ${new_candidate} + +Check Network Operational Information After Fail + [Documentation] Check device is in operational inventory and topology in all cluster instances. + Check OpenFlow Network Operational Information For One Device ${new_cluster_list} + +Add Configuration In Owner and Verify After Fail + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Add Sample Flow and Verify ${new_cluster_list} ${new_owner} + +Modify Configuration In Owner and Verify After Fail + [Documentation] Modify Flow in Owner and verify it gets applied from all instances. + Modify Sample Flow and Verify ${new_cluster_list} ${new_owner} + +Delete Configuration In Owner and Verify After Fail + [Documentation] Delete Flow in Owner and verify it gets applied from all instances. + Delete Sample Flow and Verify ${new_cluster_list} ${new_owner} + +Add Configuration In Candidate and Verify After Fail + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Add Sample Flow and Verify ${new_cluster_list} ${new_candidate} + +Modify Configuration In Candidate and Verify After Fail + [Documentation] Modify Flow in Owner and verify it gets applied from all instances. + Modify Sample Flow and Verify ${new_cluster_list} ${new_candidate} + +Delete Configuration In Candidate and Verify After Fail + [Documentation] Delete Flow in Owner and verify it gets applied from all instances. + Delete Sample Flow and Verify ${new_cluster_list} ${new_candidate} + +Send RPC Add to Owner and Verify After Fail + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Send RPC Add Sample Flow and Verify ${new_cluster_list} ${new_owner} + +Send RPC Delete to Owner and Verify After Fail + [Documentation] Delete Flow in Owner and verify it gets removed from all instances. + Send RPC Delete Sample Flow and Verify ${new_cluster_list} ${new_owner} + +Send RPC Add to Candidate and Verify After Fail + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Send RPC Add Sample Flow and Verify ${new_cluster_list} ${new_candidate} + +Send RPC Delete to Candidate and Verify After Fail + [Documentation] Delete Flow in Owner and verify it gets removed from all instances. + Send RPC Delete Sample Flow and Verify ${new_cluster_list} ${new_candidate} + +Modify Network and Verify After Fail + [Documentation] Take a link down and verify port status in all instances. + Take OpenFlow Device Link Down and Verify ${new_cluster_list} + +Restore Network and Verify After Fail + [Documentation] Take the link up and verify port status in all instances. + Take OpenFlow Device Link Up and Verify ${new_cluster_list} + +Start Old Owner Instance + [Documentation] Kill Owner Instance and verify it is dead + Start Multiple Controllers 300s ${original_owner} + +Check Shards Status After Recover + [Documentation] Create original cluster list and check Status for all shards in OpenFlow application. + Wait Until Keyword Succeeds 5s 1s Check OpenFlow Shards Status ${original_cluster_list} + +Check Entity Owner Status After Recover + [Documentation] Check Entity Owner Status and identify owner and candidate. + ${new_owner} ${new_candidates_list} Wait Until Keyword Succeeds 5s 1s Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} + Set Suite Variable ${new_owner} + +Check Network Operational Information After Recover + [Documentation] Check device is in operational inventory and topology in all cluster instances. + Check OpenFlow Network Operational Information For One Device ${original_cluster_list} + +Add Configuration In Owner and Verify After Recover + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Add Sample Flow and Verify ${original_cluster_list} ${new_owner} + +Modify Configuration In Owner and Verify After Recover + [Documentation] Modify Flow in Owner and verify it gets applied from all instances. + Modify Sample Flow and Verify ${original_cluster_list} ${new_owner} + +Delete Configuration In Owner and Verify After Recover + [Documentation] Delete Flow in Owner and verify it gets applied from all instances. + Delete Sample Flow and Verify ${original_cluster_list} ${new_owner} + +Add Configuration In Old Owner and Verify After Recover + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Add Sample Flow and Verify ${originalcluster_list} ${original_owner} + +Modify Configuration In Old Owner and Verify After Recover + [Documentation] Modify Flow in Owner and verify it gets applied from all instances. + Modify Sample Flow and Verify ${original_cluster_list} ${original_owner} + +Delete Configuration In Old Owner and Verify After Recover + [Documentation] Delete Flow in Owner and verify it gets applied from all instances. + Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + +Send RPC Add to Owner and Verify After Recover + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Send RPC Add Sample Flow and Verify ${original_cluster_list} ${new_owner} + +Send RPC Delete to Owner and Verify After Recover + [Documentation] Delete Flow in Owner and verify it gets removed from all instances. + Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${new_owner} + +Send RPC Add to Old Owner and Verify After Recover + [Documentation] Add Flow in Owner and verify it gets applied from all instances. + Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_owner} + +Send RPC Delete to Old Owner and Verify After Recover + [Documentation] Delete Flow in Owner and verify it gets removed from all instances. + Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + +Modify Network and Verify After Recover [Documentation] Take a link down and verify port status in all instances. - ${dictionary}= Create Dictionary "link-down":true=1 - ${ouput}= Send Mininet Command ${mininet_conn_id} link s1 h1 down - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_port_1} + Take OpenFlow Device Link Down and Verify ${original_cluster_list} -Take a Link Up and Verify +Restore Network and Verify After Recover [Documentation] Take the link up and verify port status in all instances. - ${dictionary}= Create Dictionary "link-down":true=0 - ${ouput}= Send Mininet Command ${mininet_conn_id} link s1 h1 up - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${operational_port_1} + Take OpenFlow Device Link Up and Verify ${original_cluster_list} Stop Mininet and Exit [Documentation] Stop mininet and exit connection. @@ -139,7 +224,4 @@ Stop Mininet and Exit Check No Network Operational Information [Documentation] Check device is not in operational inventory or topology in all cluster instances. - ${dictionary} Create Dictionary openflow:1=0 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${OPERATIONAL_NODES_API} - ${dictionary} Create Dictionary openflow:1=0 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${original_cluster_list} ${dictionary} ${OPERATIONAL_TOPO_API} + Check No OpenFlow Network Operational Information ${original_cluster_list} diff --git a/csit/testplans/openflowplugin-clustering.txt b/csit/testplans/openflowplugin-clustering.txt index b200f39fc9..42ec6a2e1f 100644 --- a/csit/testplans/openflowplugin-clustering.txt +++ b/csit/testplans/openflowplugin-clustering.txt @@ -1,3 +1,4 @@ # Place the suites in run order: integration/test/csit/suites/openflowplugin/Clustering +integration/test/csit/suites/openflowplugin/Clustering integration/test/csit/suites/openflowplugin/Sanity3Node -- 2.36.6