From: Luis Gomez Date: Wed, 24 Feb 2016 05:34:12 +0000 (-0800) Subject: Add topology check in OpenFlow cluster test X-Git-Tag: release/beryllium-sr1~51 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=integration%2Ftest.git;a=commitdiff_plain;h=da55118c501c4877e2f685b7d062b3335b5c3884 Add topology check in OpenFlow cluster test Add 2 more switches connecting to first switch candidate instance. Verify new inventory and topology during owner instance failure and recover. verify topology when link between switches goes down and up. Change-Id: I153a24599ce9478bc8e57cd161d46c606aa26bad Signed-off-by: Luis Gomez --- diff --git a/csit/libraries/ClusterKeywords.robot b/csit/libraries/ClusterKeywords.robot index 04ab7b613a..dcf9bb11b8 100644 --- a/csit/libraries/ClusterKeywords.robot +++ b/csit/libraries/ClusterKeywords.robot @@ -25,7 +25,7 @@ Create Controller Sessions ${NUM_ODL_SYSTEM}= Convert to Integer ${NUM_ODL_SYSTEM} : FOR ${i} IN RANGE ${NUM_ODL_SYSTEM} \ Log Create Session ${ODL_SYSTEM_${i+1}_IP} - \ Create Session controller${i+1} http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT} auth=${AUTH} + \ RequestsLibrary.Create Session controller${i+1} http://${ODL_SYSTEM_${i+1}_IP}:${RESTCONFPORT} auth=${AUTH} Get Cluster Shard Status [Arguments] ${controller_index_list} ${shard_type} ${shard} @@ -38,9 +38,9 @@ Get Cluster Shard Status ${leader}= Set Variable 0 ${follower_list}= Create List : FOR ${i} IN @{controller_index_list} - \ ${data}= Get Data From URI controller${i} ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type} + \ ${data}= Utils.Get Data From URI controller${i} ${jolokia_read}:Category=Shards,name=member-${i}-shard-${shard}-${shard_type},type=${type} \ Log ${data} - \ ${json}= To Json ${data} + \ ${json}= RequestsLibrary.To Json ${data} \ ${status}= Get From Dictionary &{json}[value] RaftState \ Log Controller ${ODL_SYSTEM_${i}_IP} is ${status} for shard ${shard} \ Run Keyword If '${status}' == 'Leader' Set Test Variable ${leader} ${i} @@ -54,9 +54,9 @@ Check Item Occurrence At URI In Cluster [Documentation] Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list} ... and check for occurrences of items expressed in a dictionary ${dictionary_item_occurrence}. : FOR ${i} IN @{controller_index_list} - \ ${data} Get Data From URI controller${i} ${uri} + \ ${data} Utils.Get Data From URI controller${i} ${uri} \ Log ${data} - \ Check Item Occurrence ${data} ${dictionary_item_occurrence} + \ Utils.Check Item Occurrence ${data} ${dictionary_item_occurrence} Put And Check At URI In Cluster [Arguments] ${controller_index_list} ${controller_index} ${uri} ${body} @@ -89,15 +89,15 @@ Kill Multiple Controllers [Arguments] @{controller_index_list} [Documentation] Give this keyword a scalar or list of controllers to be stopped. : FOR ${i} IN @{controller_index_list} - \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh - \ Controller Down Check ${ODL_SYSTEM_${i}_IP} + \ ${output}= Utils.Run Command On Controller ${ODL_SYSTEM_${i}_IP} ps axf | grep karaf | grep -v grep | awk '{print \"kill -9 \" $1}' | sh + \ ClusterKeywords.Controller Down Check ${ODL_SYSTEM_${i}_IP} Start Multiple Controllers [Arguments] ${timeout} @{controller_index_list} [Documentation] Give this keyword a scalar or list of controllers to be started. : FOR ${i} IN @{controller_index_list} - \ ${output}= Run Command On Controller ${ODL_SYSTEM_${i}_IP} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start - \ Wait For Controller Sync ${timeout} ${ODL_SYSTEM_${i}_IP} + \ ${output}= Utils.Run Command On Controller ${ODL_SYSTEM_${i}_IP} ${WORKSPACE}/${BUNDLEFOLDER}/bin/start + \ ClusterKeywords.Wait For Controller Sync ${timeout} ${ODL_SYSTEM_${i}_IP} Get Controller List [Arguments] ${exclude_controller}=${EMPTY} diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot index 4aaa55c379..b96fc1c9b2 100644 --- a/csit/libraries/ClusterOpenFlow.robot +++ b/csit/libraries/ClusterOpenFlow.robot @@ -15,10 +15,10 @@ ${operational_port_1} ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector Check OpenFlow Shards Status [Arguments] ${controller_index_list} [Documentation] Check Status for all shards in OpenFlow application. - ${inv_conf_leader} ${inv_conf_followers_list} Get Cluster Shard Status ${controller_index_list} config inventory - ${inv_oper_leader} ${inv_oper_followers_list} Get Cluster Shard Status ${controller_index_list} operational inventory - ${topo_oper_leader} ${topo_oper_followers_list} Get Cluster Shard Status ${controller_index_list} operational topology - ${owner_oper_leader} ${owner_oper_followers_list} Get Cluster Shard Status ${controller_index_list} operational entity-ownership + ${inv_conf_leader} ${inv_conf_followers_list} ClusterKeywords.Get Cluster Shard Status ${controller_index_list} config inventory + ${inv_oper_leader} ${inv_oper_followers_list} ClusterKeywords.Get Cluster Shard Status ${controller_index_list} operational inventory + ${topo_oper_leader} ${topo_oper_followers_list} ClusterKeywords.Get Cluster Shard Status ${controller_index_list} operational topology + ${owner_oper_leader} ${owner_oper_followers_list} ClusterKeywords.Get Cluster Shard Status ${controller_index_list} operational entity-ownership Log config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list} Log operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list} Log operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list} @@ -27,7 +27,7 @@ Check OpenFlow Shards Status Check OpenFlow Shards Status After Cluster Event [Arguments] ${controller_index_list} [Documentation] Check Shards Status after some cluster event. - Wait Until Keyword Succeeds 90s 1s Check OpenFlow Shards Status ${controller_index_list} + Wait Until Keyword Succeeds 90s 1s ClusterOpenFlow.Check OpenFlow Shards Status ${controller_index_list} Get Cluster Entity Owner For Openflow [Arguments] ${controller_index_list} ${device_type} ${device} @@ -35,17 +35,17 @@ Get Cluster Entity Owner For Openflow ... ${device_type} is normally openflow. ${length}= Get Length ${controller_index_list} ${candidates_list}= Create List - ${data}= Get Data From URI controller@{controller_index_list}[0] /restconf/operational/entity-owners:entity-owners + ${data}= Utils.Get Data From URI controller@{controller_index_list}[0] /restconf/operational/entity-owners:entity-owners Log ${data} ${data}= Replace String ${data} /general-entity:entity[general-entity:name=' ${EMPTY} ${clear_data}= Replace String ${data} '] ${EMPTY} Log ${clear_data} - ${json}= To Json ${clear_data} + ${json}= RequestsLibrary.To Json ${clear_data} ${entity_type_list}= Get From Dictionary &{json}[entity-owners] entity-type ${entity_type_index}= Get Index From List Of Dictionaries ${entity_type_list} type ${device_type} Should Not Be Equal ${entity_type_index} -1 No Entity Owner found for ${device_type} ${entity_list}= Get From Dictionary @{entity_type_list}[${entity_type_index}] entity - ${entity_index}= Get Index From List Of Dictionaries ${entity_list} id ${device} + ${entity_index}= Utils.Get Index From List Of Dictionaries ${entity_list} id ${device} Should Not Be Equal ${entity_index} -1 Device ${device} not found in Entity Owner ${device_type} ${entity_owner}= Get From Dictionary @{entity_list}[${entity_index}] owner Should Not Be Empty ${entity_owner} No owner found for ${device} @@ -63,28 +63,29 @@ Get Cluster Entity Owner For Openflow [Return] ${owner} ${candidates_list} Get OpenFlow Entity Owner Status For One Device - [Arguments] ${controller_index_list} + [Arguments] ${controller_index_list} ${device} [Documentation] Check Entity Owner Status and identify owner and candidate. - ${owner} ${candidates_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Entity Owner For Openflow ${controller_index_list} - ... openflow openflow:1 + ${owner} ${candidates_list} Wait Until Keyword Succeeds 10s 1s ClusterOpenFlow.Get Cluster Entity Owner For Openflow ${controller_index_list} + ... openflow ${device} [Return] ${owner} ${candidates_list} -Check OpenFlow Network Operational Information For One Device +Check OpenFlow Network Operational Information For Sample Topology [Arguments] ${controller_index_list} - [Documentation] Check device openflow:1 is in operational inventory and topology in all instances in ${controller_index_list}. - ... Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector. - ${dictionary} Create Dictionary openflow:1=4 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_NODES_API} - ${dictionary} Create Dictionary openflow:1=11 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} + [Documentation] Check devices in tree,2 are in operational inventory and topology in all instances in ${controller_index_list}. + ... Inventory should show 1x node_id per device 1x node_id per connector. Topology should show 2x node_id per device + 3x node_id per connector + ... + 5x node_id per link termination. TODO: A Keyword that can calculate this based on mininet topology. + ${dictionary} Create Dictionary openflow:1=4 openflow:2=5 openflow:3=5 + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_NODES_API} + ${dictionary} Create Dictionary openflow:1=21 openflow:2=19 openflow:3=19 + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} Check No OpenFlow Network Operational Information [Arguments] ${controller_index_list} [Documentation] Check device is not in operational inventory or topology in all cluster instances in ${controller_index_list}. ${dictionary} Create Dictionary openflow=0 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_NODES_API} + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_NODES_API} ${dictionary} Create Dictionary openflow=0 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} Add Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} @@ -94,8 +95,8 @@ Add Sample Flow And Verify # replicating some of the matches in the flows section. Same comment applies for further keywords. Run Keyword If '${ODL_OF_PLUGIN}' == 'helium' Set Test Variable &{dictionary} 10.0.1.0/24=2 "output-node-connector":"1"=1 Run Keyword If '${ODL_OF_PLUGIN}' == 'lithium' Set Test Variable &{dictionary} 10.0.1.0/24=1 "output-node-connector":"1"=1 - Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} - Wait Until Keyword Succeeds 15s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + ClusterKeywords.Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} + Wait Until Keyword Succeeds 15s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Modify Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} @@ -103,15 +104,15 @@ Modify Sample Flow And Verify ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/sample_flow_2.json Run Keyword If '${ODL_OF_PLUGIN}' == 'helium' Set Test Variable &{dictionary} 10.0.1.0/24=2 "output-node-connector":"2"=1 Run Keyword If '${ODL_OF_PLUGIN}' == 'lithium' Set Test Variable &{dictionary} 10.0.1.0/24=1 "output-node-connector":"2"=1 - Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} - Wait Until Keyword Succeeds 15s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + ClusterKeywords.Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} + Wait Until Keyword Succeeds 15s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Delete Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} [Documentation] Delete sample flow in Owner and verify it gets removed from all instances. ${dictionary}= Create Dictionary 10.0.2.0/24=0 - Delete And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + ClusterKeywords.Delete And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Send RPC Add Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} @@ -122,7 +123,7 @@ Send RPC Add Sample Flow And Verify ${resp} RequestsLibrary.Post Request controller${controller_index} /restconf/operations/sal-flow:add-flow data=${body} headers=${HEADERS_YANG_JSON} Log ${resp.content} Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 15s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + Wait Until Keyword Succeeds 15s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Send RPC Delete Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} @@ -132,18 +133,22 @@ Send RPC Delete Sample Flow And Verify ${resp} RequestsLibrary.Post Request controller${controller_index} /restconf/operations/sal-flow:remove-flow data=${body} headers=${HEADERS_YANG_JSON} Log ${resp.content} Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Take OpenFlow Device Link Down and Verify [Arguments] ${controller_index_list} [Documentation] Take a link down and verify port status in all instances in ${controller_index_list}. ${dictionary}= Create Dictionary "link-down":true=1 - ${ouput}= Send Mininet Command ${mininet_conn_id} link s1 h1 down - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_port_1} + ${ouput}= MininetKeywords.Send Mininet Command ${mininet_conn_id} link s1 s2 down + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_port_1} + ${dictionary} Create Dictionary openflow:1=16 openflow:2=14 openflow:3=19 + Wait Until Keyword Succeeds 20s 2s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} Take OpenFlow Device Link Up and Verify [Arguments] ${controller_index_list} [Documentation] Take the link up and verify port status in all instances in ${controller_index_list}. ${dictionary}= Create Dictionary "link-down":true=0 - ${ouput}= Send Mininet Command ${mininet_conn_id} link s1 h1 up - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_port_1} + ${ouput}= MininetKeywords.Send Mininet Command ${mininet_conn_id} link s1 s2 up + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_port_1} + ${dictionary} Create Dictionary openflow:1=21 openflow:2=19 openflow:3=19 + Wait Until Keyword Succeeds 5s 1s ClusterKeywords.Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${OPERATIONAL_TOPO_API} diff --git a/csit/libraries/MininetKeywords.robot b/csit/libraries/MininetKeywords.robot index ef3e440985..828fe3f66c 100644 --- a/csit/libraries/MininetKeywords.robot +++ b/csit/libraries/MininetKeywords.robot @@ -2,6 +2,7 @@ Documentation Mininet library. This library is useful for tests using mininet tool to simulate devices. Library SSHLibrary Resource Utils.robot +Resource OVSDB.robot Variables ../variables/Variables.py *** Keywords *** @@ -9,15 +10,15 @@ Start Mininet Single Controller [Arguments] ${mininet}=${TOOLS_SYSTEM_IP} ${controller}=${ODL_SYSTEM_IP} ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13 ${custom}=${EMPTY} ${ofport}=6633 [Documentation] Start Mininet with custom topology and connect to controller. Log Clear any existing mininet - Clean Mininet System ${mininet} - ${mininet_conn_id}= Open Connection ${mininet} prompt=${TOOLS_SYSTEM_PROMPT} timeout=${DEFAULT_TIMEOUT} + Utils.Clean Mininet System ${mininet} + ${mininet_conn_id}= SSHLibrary.Open Connection ${mininet} prompt=${TOOLS_SYSTEM_PROMPT} timeout=${DEFAULT_TIMEOUT} Set Suite Variable ${mininet_conn_id} - Flexible Mininet Login + Utils.Flexible Mininet Login Run Keyword If '${custom}' != '${EMPTY}' Put File ${custom} Log Start mininet ${options} to ${controller} - Write sudo mn --controller 'remote,ip=${controller},port=${ofport}' ${options} - Read Until mininet> - ${output}= Run Command On Mininet ${mininet} sudo ovs-vsctl show + SSHLibrary.Write sudo mn --controller 'remote,ip=${controller},port=${ofport}' ${options} + SSHLibrary.Read Until mininet> + ${output}= Utils.Run Command On Mininet ${mininet} sudo ovs-vsctl show Log ${output} [Return] ${mininet_conn_id} @@ -25,56 +26,37 @@ Start Mininet Multiple Controllers [Arguments] ${mininet} ${controller_index_list} ${options}=--topo tree,1 --switch ovsk,protocols=OpenFlow13 ${custom}=${EMPTY} ${ofport}=6633 [Documentation] Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}. Log Clear any existing mininet - Clean Mininet System ${mininet} - ${mininet_conn_id}= Open Connection ${mininet} prompt=${TOOLS_SYSTEM_PROMPT} timeout=${DEFAULT_TIMEOUT} + Utils.Clean Mininet System ${mininet} + ${mininet_conn_id}= SSHLibrary.Open Connection ${mininet} prompt=${TOOLS_SYSTEM_PROMPT} timeout=${DEFAULT_TIMEOUT} Set Suite Variable ${mininet_conn_id} - Flexible Mininet Login + Utils.Flexible Mininet Login Run Keyword If '${custom}' != '${EMPTY}' Put File ${custom} Log Start mininet ${options} - Write sudo mn ${options} - Read Until mininet> + SSHLibrary.Write sudo mn ${options} + SSHLibrary.Read Until mininet> Log Create controller configuration - ${ovs_opt}= Set Variable + ${controller_opt}= Set Variable : FOR ${index} IN @{controller_index_list} - \ ${ovs_opt}= Catenate ${ovs_opt} ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ofport} - \ Log ${ovs_opt} + \ ${controller_opt}= Catenate ${controller_opt} ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ofport} + \ Log ${controller_opt} Log Find Number of OVS bridges - ${num_bridges} Run Command On Mininet ${mininet} sudo ovs-vsctl show | grep Bridge | wc -l + ${num_bridges} Utils.Run Command On Mininet ${mininet} sudo ovs-vsctl show | grep Bridge | wc -l ${num_bridges}= Convert To Integer ${num_bridges} - Log Configure OVS controllers ${ovs_opt} in all bridges + Log Configure OVS controllers ${controller_opt} in all bridges : FOR ${i} IN RANGE 1 ${num_bridges+1} - \ ${bridge}= Run Command On Mininet ${mininet} sudo ovs-vsctl show | grep Bridge | cut -c 12- | sort | head -${i} | tail -1 - \ Run Command On Mininet ${mininet} sudo ovs-vsctl set-controller ${bridge} ${ovs_opt} + \ ${bridge}= Utils.Run Command On Mininet ${mininet} sudo ovs-vsctl show | grep Bridge | cut -c 12- | sort | head -${i} | tail -1 + \ OVSDB.Set Controller In OVS Bridge ${mininet} ${bridge} ${controller_opt} Log Check OVS configuratiom - ${output}= Run Command On Mininet ${mininet} sudo ovs-vsctl show - Log ${output} - [Return] ${mininet_conn_id} - -Add Multiple Managers to OVS - [Arguments] ${mininet} ${controller_index_list} ${ovs_mgr_port}=6640 - [Documentation] Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}. - Log Clear any existing mininet - Clean Mininet System ${mininet} - ${mininet_conn_id}= Open Connection ${mininet} prompt=${TOOLS_SYSTEM_PROMPT} timeout=${DEFAULT_TIMEOUT} - Set Suite Variable ${mininet_conn_id} - Flexible Mininet Login - ${ovs_opt}= Set Variable - : FOR ${index} IN @{controller_index_list} - \ ${ovs_opt}= Catenate ${ovs_opt} ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ovs_mgr_port} - \ Log ${ovs_opt} - Log Configure OVS Managers in the OVS - Run Command On Mininet ${mininet} sudo ovs-vsctl set-manager ${ovs_opt} - Log Check OVS configuratiom - ${output}= Run Command On Mininet ${mininet} sudo ovs-vsctl show + ${output}= Utils.Run Command On Mininet ${mininet} sudo ovs-vsctl show Log ${output} [Return] ${mininet_conn_id} Send Mininet Command [Arguments] ${mininet_conn_id} ${cmd}=help [Documentation] Sends Command ${cmd} to Mininet session ${mininet_conn_id} and returns read buffer response. - Switch Connection ${mininet_conn_id} + SSHLibrary.Switch Connection ${mininet_conn_id} SSHLibrary.Write ${cmd} - ${output}= Read Until mininet> + ${output}= SSHLibrary.Read Until mininet> [Return] ${output} Send Mininet Command Multiple Sessions @@ -82,20 +64,20 @@ Send Mininet Command Multiple Sessions [Documentation] Sends Command ${cmd} to Mininet sessions in ${mininet_conn_list} and returns list of read buffer responses. ${output_list}= Create List : FOR ${mininet_conn_id} IN @{mininet_conn_list} - \ ${output}= Send Mininet Command ${mininet_conn_id} ${cmd} + \ ${output}= Utils.Send Mininet Command ${mininet_conn_id} ${cmd} \ Append To List ${output_list} ${output} [Return] ${output_list} Stop Mininet And Exit [Arguments] ${mininet_conn_id} [Documentation] Stops Mininet and exits session ${mininet_conn_id} - Switch Connection ${mininet_conn_id} + SSHLibrary.Switch Connection ${mininet_conn_id} SSHLibrary.Write exit - Read Until ${TOOLS_SYSTEM_PROMPT} + SSHLibrary.Read Until ${TOOLS_SYSTEM_PROMPT} Close Connection Stop Mininet And Exit Multiple Sessions [Arguments] ${mininet_conn_list} [Documentation] Stops Mininet and exits sessions in ${mininet_conn_list}. : FOR ${mininet_conn_id} IN @{mininet_conn_list} - \ Stop Mininet And Exit ${mininet_conn_id} + \ MininetKeywords.Stop Mininet And Exit ${mininet_conn_id} diff --git a/csit/libraries/OVSDB.robot b/csit/libraries/OVSDB.robot index ec50a5a8f7..d4b4fb3994 100644 --- a/csit/libraries/OVSDB.robot +++ b/csit/libraries/OVSDB.robot @@ -109,3 +109,29 @@ Clean OVSDB Test Environment Run Command On Mininet ${tools_system} sudo /usr/share/openvswitch/scripts/ovs-ctl stop Run Command On Mininet ${tools_system} sudo rm -rf /etc/openvswitch/conf.db Run Command On Mininet ${tools_system} sudo /usr/share/openvswitch/scripts/ovs-ctl start + +Set Controller In OVS Bridge + [Arguments] ${mininet} ${bridge} ${controller_opt} + [Documentation] Sets controller for a given OVS ${bridge} using controller options in ${controller_opt} + Run Command On Mininet ${mininet} sudo ovs-vsctl del-controller ${bridge} + Run Command On Mininet ${mininet} sudo ovs-vsctl set-controller ${bridge} ${controller_opt} + +Add Multiple Managers to OVS + [Arguments] ${mininet} ${controller_index_list} ${ovs_mgr_port}=6640 + [Documentation] Start Mininet with custom topology and connect to all controllers in the ${controller_index_list}. + Log Clear any existing mininet + Clean Mininet System ${mininet} + ${mininet_conn_id}= Open Connection ${mininet} prompt=${TOOLS_SYSTEM_PROMPT} timeout=${DEFAULT_TIMEOUT} + Set Suite Variable ${mininet_conn_id} + Flexible Mininet Login + ${ovs_opt}= Set Variable + : FOR ${index} IN @{controller_index_list} + \ ${ovs_opt}= Catenate ${ovs_opt} ${SPACE}tcp:${ODL_SYSTEM_${index}_IP}:${ovs_mgr_port} + \ Log ${ovs_opt} + Log Configure OVS Managers in the OVS + Run Command On Mininet ${mininet} sudo ovs-vsctl set-manager ${ovs_opt} + Log Check OVS configuratiom + ${output}= Run Command On Mininet ${mininet} sudo ovs-vsctl show + Log ${output} + [Return] ${mininet_conn_id} + diff --git a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot index 3454295405..29174e4c8a 100644 --- a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot +++ b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot @@ -11,220 +11,230 @@ Variables ../../../variables/Variables.py *** Test Cases *** Create Original Cluster List [Documentation] Create original cluster list. - ${original_cluster_list} Create Controller Index List + ${original_cluster_list} ClusterKeywords.Create Controller Index List Set Suite Variable ${original_cluster_list} Check Shards Status Before Fail [Documentation] Check Status for all shards in OpenFlow application. - Check OpenFlow Shards Status ${original_cluster_list} + ClusterOpenFlow.Check OpenFlow Shards Status ${original_cluster_list} Start Mininet Multiple Connections - [Documentation] Start mininet with connection to all cluster instances. - ${mininet_conn_id}= Start Mininet Multiple Controllers ${TOOLS_SYSTEM_IP} ${original_cluster_list} + [Documentation] Start mininet tree,2 with connection to all cluster instances. + ${mininet_conn_id}= MininetKeywords.Start Mininet Multiple Controllers ${TOOLS_SYSTEM_IP} ${original_cluster_list} --topo tree,2 --switch ovsk,protocols=OpenFlow13 Set Suite Variable ${mininet_conn_id} Check Entity Owner Status And Find Owner and Candidate Before Fail - [Documentation] Check Entity Owner Status and identify owner and candidate. - ${original_owner} ${original_candidates_list} Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} + [Documentation] Check Entity Owner Status and identify owner and candidate for first switch s1. + ${original_owner} ${original_candidates_list} ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} openflow:1 ${original_candidate}= Get From List ${original_candidates_list} 0 Set Suite Variable ${original_owner} Set Suite Variable ${original_candidate} +Reconnect Extra Switches To Candidate And Check Entity Owner + [Documentation] Connect switches s2 and s3 to candidate instance. + OVSDB.Set Controller In OVS Bridge ${TOOLS_SYSTEM_IP} s2 tcp:${ODL_SYSTEM_${original_candidate}_IP}:6633 + OVSDB.Set Controller In OVS Bridge ${TOOLS_SYSTEM_IP} s3 tcp:${ODL_SYSTEM_${original_candidate}_IP}:6633 + ${original_candidate_list}= Create List ${original_candidate} + ${owner} ${candidates_list} ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device ${original_candidate_list} openflow:2 + Should Be Equal ${owner} ${original_candidate} + ${owner} ${candidates_list} ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device ${original_candidate_list} openflow:3 + Should Be Equal ${owner} ${original_candidate} + Check Network Operational Information Before Fail - [Documentation] Check device is in operational inventory and topology in all cluster instances. - Check OpenFlow Network Operational Information For One Device ${original_cluster_list} + [Documentation] Check devices in operational inventory and topology in all cluster instances. + ClusterOpenFlow.Check OpenFlow Network Operational Information For Sample Topology ${original_cluster_list} Add Configuration In Owner and Verify Before Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Add Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Add Sample Flow and Verify ${original_cluster_list} ${original_owner} Modify Configuration In Owner and Verify Before Fail [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - Modify Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Modify Sample Flow and Verify ${original_cluster_list} ${original_owner} Delete Configuration In Owner and Verify Before Fail [Documentation] Delete Flow in Owner and verify it gets applied from all instances. - Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} Add Configuration In Candidate and Verify Before Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Add Sample Flow and Verify ${original_cluster_list} ${original_candidate} + ClusterOpenFlow.Add Sample Flow and Verify ${original_cluster_list} ${original_candidate} Modify Configuration In Candidate and Verify Before Fail [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - Modify Sample Flow and Verify ${original_cluster_list} ${original_candidate} + ClusterOpenFlow.Modify Sample Flow and Verify ${original_cluster_list} ${original_candidate} Delete Configuration In Candidate and Verify Before Fail [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - Delete Sample Flow and Verify ${original_cluster_list} ${original_candidate} + ClusterOpenFlow.Delete Sample Flow and Verify ${original_cluster_list} ${original_candidate} Send RPC Add to Owner and Verify Before Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_owner} Send RPC Delete to Owner and Verify Before Fail [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} Send RPC Add to Candidate and Verify Before Fail [Documentation] Add Flow in Candidate and verify it gets applied from all instances. - Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_candidate} + ClusterOpenFlow.Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_candidate} Send RPC Delete to Candidate and Verify Before Fail [Documentation] Delete Flow in Candidate and verify it gets removed from all instances. - Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_candidate} + ClusterOpenFlow.Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_candidate} Modify Network And Verify Before Fail [Documentation] Take a link down and verify port status in all instances. - Take OpenFlow Device Link Down and Verify ${original_cluster_list} + ClusterOpenFlow.Take OpenFlow Device Link Down and Verify ${original_cluster_list} Restore Network And Verify Before Fail [Documentation] Take the link up and verify port status in all instances. - Take OpenFlow Device Link Up and Verify ${original_cluster_list} + ClusterOpenFlow.Take OpenFlow Device Link Up and Verify ${original_cluster_list} Kill Owner Instance [Documentation] Kill Owner Instance and verify it is dead - Kill Multiple Controllers ${original_owner} + ClusterKeywords.Kill Multiple Controllers ${original_owner} ${new_cluster_list} Create Controller Index List Remove Values From List ${new_cluster_list} ${original_owner} Set Suite Variable ${new_cluster_list} Check Shards Status After Fail [Documentation] Create original cluster list and check Status for all shards in OpenFlow application. - Check OpenFlow Shards Status After Cluster Event ${new_cluster_list} + ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event ${new_cluster_list} Check Entity Owner Status And Find Owner and Candidate After Fail [Documentation] Check Entity Owner Status and identify owner and candidate. - ${new_owner} ${new_candidates_list} Get OpenFlow Entity Owner Status For One Device ${new_cluster_list} + ${new_owner} ${new_candidates_list} ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device ${new_cluster_list} openflow:1 Run Keyword And Continue On Failure List Should Not Contain Value ${new_candidates_list} ${original_owner} Original owner ${original_owner} still in candidate list. Remove Values From List ${new_candidates_list} ${original_owner} ${new_candidate}= Get From List ${new_candidates_list} 0 Set Suite Variable ${new_owner} Set Suite Variable ${new_candidate} - [Teardown] Report_Failure_Due_To_Bug 5004 + [Teardown] Utils.Report_Failure_Due_To_Bug 5004 Check Network Operational Information After Fail - [Documentation] Check device is in operational inventory and topology in all cluster instances. - Check OpenFlow Network Operational Information For One Device ${new_cluster_list} + [Documentation] Check devices in operational inventory and topology in all cluster instances. + ClusterOpenFlow.Check OpenFlow Network Operational Information For Sample Topology ${new_cluster_list} Add Configuration In Owner and Verify After Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Add Sample Flow and Verify ${new_cluster_list} ${new_owner} + ClusterOpenFlow.Add Sample Flow and Verify ${new_cluster_list} ${new_owner} Modify Configuration In Owner and Verify After Fail [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - Modify Sample Flow and Verify ${new_cluster_list} ${new_owner} + ClusterOpenFlow.Modify Sample Flow and Verify ${new_cluster_list} ${new_owner} Delete Configuration In Owner and Verify After Fail [Documentation] Delete Flow in Owner and verify it gets applied from all instances. - Delete Sample Flow and Verify ${new_cluster_list} ${new_owner} + ClusterOpenFlow.Delete Sample Flow and Verify ${new_cluster_list} ${new_owner} Add Configuration In Candidate and Verify After Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Add Sample Flow and Verify ${new_cluster_list} ${new_candidate} + ClusterOpenFlow.Add Sample Flow and Verify ${new_cluster_list} ${new_candidate} Modify Configuration In Candidate and Verify After Fail [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - Modify Sample Flow and Verify ${new_cluster_list} ${new_candidate} + ClusterOpenFlow.Modify Sample Flow and Verify ${new_cluster_list} ${new_candidate} Delete Configuration In Candidate and Verify After Fail [Documentation] Delete Flow in Owner and verify it gets applied from all instances. - Delete Sample Flow and Verify ${new_cluster_list} ${new_candidate} + ClusterOpenFlow.Delete Sample Flow and Verify ${new_cluster_list} ${new_candidate} Send RPC Add to Owner and Verify After Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Send RPC Add Sample Flow and Verify ${new_cluster_list} ${new_owner} + ClusterOpenFlow.Send RPC Add Sample Flow and Verify ${new_cluster_list} ${new_owner} Send RPC Delete to Owner and Verify After Fail [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - Send RPC Delete Sample Flow and Verify ${new_cluster_list} ${new_owner} + ClusterOpenFlow.Send RPC Delete Sample Flow and Verify ${new_cluster_list} ${new_owner} Send RPC Add to Candidate and Verify After Fail [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Send RPC Add Sample Flow and Verify ${new_cluster_list} ${new_candidate} + ClusterOpenFlow.Send RPC Add Sample Flow and Verify ${new_cluster_list} ${new_candidate} Send RPC Delete to Candidate and Verify After Fail [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - Send RPC Delete Sample Flow and Verify ${new_cluster_list} ${new_candidate} + ClusterOpenFlow.Send RPC Delete Sample Flow and Verify ${new_cluster_list} ${new_candidate} Modify Network and Verify After Fail [Documentation] Take a link down and verify port status in all instances. - Take OpenFlow Device Link Down and Verify ${new_cluster_list} + ClusterOpenFlow.Take OpenFlow Device Link Down and Verify ${new_cluster_list} Restore Network and Verify After Fail [Documentation] Take the link up and verify port status in all instances. - Take OpenFlow Device Link Up and Verify ${new_cluster_list} + ClusterOpenFlow.Take OpenFlow Device Link Up and Verify ${new_cluster_list} Start Old Owner Instance [Documentation] Start old Owner Instance and verify it is up - Start Multiple Controllers 300s ${original_owner} + ClusterKeywords.Start Multiple Controllers 300s ${original_owner} Check Shards Status After Recover [Documentation] Create original cluster list and check Status for all shards in OpenFlow application. - Check OpenFlow Shards Status After Cluster Event ${original_cluster_list} + ClusterOpenFlow.Check OpenFlow Shards Status After Cluster Event ${original_cluster_list} Check Entity Owner Status After Recover [Documentation] Check Entity Owner Status and identify owner and candidate. - ${new_owner} ${new_candidates_list} Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} + ${new_owner} ${new_candidates_list} ClusterOpenFlow.Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} openflow:1 Set Suite Variable ${new_owner} Check Network Operational Information After Recover - [Documentation] Check device is in operational inventory and topology in all cluster instances. - Check OpenFlow Network Operational Information For One Device ${original_cluster_list} + [Documentation] Check devices in operational inventory and topology in all cluster instances. + ClusterOpenFlow.Check OpenFlow Network Operational Information For Sample Topology ${original_cluster_list} Add Configuration In Owner and Verify After Recover [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Add Sample Flow and Verify ${original_cluster_list} ${new_owner} + ClusterOpenFlow.Add Sample Flow and Verify ${original_cluster_list} ${new_owner} Modify Configuration In Owner and Verify After Recover [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - Modify Sample Flow and Verify ${original_cluster_list} ${new_owner} + ClusterOpenFlow.Modify Sample Flow and Verify ${original_cluster_list} ${new_owner} Delete Configuration In Owner and Verify After Recover [Documentation] Delete Flow in Owner and verify it gets applied from all instances. - Delete Sample Flow and Verify ${original_cluster_list} ${new_owner} + ClusterOpenFlow.Delete Sample Flow and Verify ${original_cluster_list} ${new_owner} Add Configuration In Old Owner and Verify After Recover [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Add Sample Flow and Verify ${originalcluster_list} ${original_owner} + ClusterOpenFlow.Add Sample Flow and Verify ${originalcluster_list} ${original_owner} Modify Configuration In Old Owner and Verify After Recover [Documentation] Modify Flow in Owner and verify it gets applied from all instances. - Modify Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Modify Sample Flow and Verify ${original_cluster_list} ${original_owner} Delete Configuration In Old Owner and Verify After Recover [Documentation] Delete Flow in Owner and verify it gets applied from all instances. - Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} Send RPC Add to Owner and Verify After Recover [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Send RPC Add Sample Flow and Verify ${original_cluster_list} ${new_owner} + ClusterOpenFlow.Send RPC Add Sample Flow and Verify ${original_cluster_list} ${new_owner} Send RPC Delete to Owner and Verify After Recover [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${new_owner} + ClusterOpenFlow.Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${new_owner} Send RPC Add to Old Owner and Verify After Recover [Documentation] Add Flow in Owner and verify it gets applied from all instances. - Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Send RPC Add Sample Flow and Verify ${original_cluster_list} ${original_owner} Send RPC Delete to Old Owner and Verify After Recover [Documentation] Delete Flow in Owner and verify it gets removed from all instances. - Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} + ClusterOpenFlow.Send RPC Delete Sample Flow and Verify ${original_cluster_list} ${original_owner} Modify Network and Verify After Recover [Documentation] Take a link down and verify port status in all instances. - Take OpenFlow Device Link Down and Verify ${original_cluster_list} + ClusterOpenFlow.Take OpenFlow Device Link Down and Verify ${original_cluster_list} Restore Network and Verify After Recover [Documentation] Take the link up and verify port status in all instances. - Take OpenFlow Device Link Up and Verify ${original_cluster_list} + ClusterOpenFlow.Take OpenFlow Device Link Up and Verify ${original_cluster_list} Stop Mininet and Exit [Documentation] Stop mininet and exit connection. - Stop Mininet And Exit ${mininet_conn_id} - Clean Mininet System + MininetKeywords.Stop Mininet And Exit ${mininet_conn_id} + Utils.Clean Mininet System Check No Network Operational Information [Documentation] Check device is not in operational inventory or topology in all cluster instances. - Check No OpenFlow Network Operational Information ${original_cluster_list} + ClusterOpenFlow.Check No OpenFlow Network Operational Information ${original_cluster_list}