From e9b687cb4c8ad74d10b70fab3fe5de5e1b1ae0f3 Mon Sep 17 00:00:00 2001 From: Luis Gomez Date: Mon, 18 Jan 2016 16:14:57 -0800 Subject: [PATCH] Refactor OF cluster test - Move entity owner keyword to OpenFlowCluster library (plugin specific) - Add entity-owner shard check - Increase timers so we know when there is delay or real failure - Continue test when entity owner does not update list of candidates - Add support for He and Li plugins Change-Id: I0c8e87ed52d33771402e75ab8f1b18f7df9127c5 Signed-off-by: Luis Gomez --- csit/libraries/ClusterKeywords.robot | 32 --------- csit/libraries/ClusterOpenFlow.robot | 66 +++++++++++++++---- .../010__Cluster_HA_Owner_Failover.robot | 11 ++-- ...penflowplugin-clustering-helium-design.txt | 4 ++ csit/variables/Variables.py | 3 + 5 files changed, 67 insertions(+), 49 deletions(-) create mode 100644 csit/testplans/openflowplugin-clustering-helium-design.txt diff --git a/csit/libraries/ClusterKeywords.robot b/csit/libraries/ClusterKeywords.robot index 7e94198f89..5f19dff7fd 100644 --- a/csit/libraries/ClusterKeywords.robot +++ b/csit/libraries/ClusterKeywords.robot @@ -49,38 +49,6 @@ Get Cluster Shard Status Length Should Be ${follower_list} ${lenght-1} Not enough or too many Followers in shard ${shard_type} ${shard} [Return] ${leader} ${follower_list} -Get Cluster Entity Owner Status - [Arguments] ${controller_index_list} ${device_type} ${device} - [Documentation] Checks Entity Owner status for a ${device} and returns owner index and list of candidates from a ${controller_index_list}. - ... ${device_type} is openflow, ovsdb, etc... - ${length}= Get Length ${controller_index_list} - ${candidates_list}= Create List - ${data}= Get Data From URI controller@{controller_index_list}[0] /restconf/operational/entity-owners:entity-owners - Log ${data} - ${data}= Replace String ${data} /general-entity:entity[general-entity:name=' ${EMPTY} - ${clear_data}= Replace String ${data} '] ${EMPTY} - Log ${clear_data} - ${json}= To Json ${clear_data} - ${entity_type_list}= Get From Dictionary &{json}[entity-owners] entity-type - ${entity_type_index}= Get Index From List Of Dictionaries ${entity_type_list} type ${device_type} - Should Not Be Equal ${entity_type_index} -1 No Entity Owner found for ${device_type} - ${entity_list}= Get From Dictionary @{entity_type_list}[${entity_type_index}] entity - ${entity_index}= Get Index From List Of Dictionaries ${entity_list} id ${device} - Should Not Be Equal ${entity_index} -1 Device ${device} not found in Entity Owner ${device_type} - ${entity_owner}= Get From Dictionary @{entity_list}[${entity_index}] owner - Should Not Be Empty ${entity_owner} No owner found for ${device} - ${owner}= Replace String ${entity_owner} member- ${EMPTY} - ${owner}= Convert To Integer ${owner} - List Should Contain Value ${controller_index_list} ${owner} Owner ${owner} not exisiting in ${controller_index_list} - ${entity_candidates_list}= Get From Dictionary @{entity_list}[${entity_index}] candidate - ${list_length}= Get Length ${entity_candidates_list} - : FOR ${entity_candidate} IN @{entity_candidates_list} - \ ${candidate}= Replace String &{entity_candidate}[name] member- ${EMPTY} - \ ${candidate}= Convert To Integer ${candidate} - \ List Should Contain Value ${controller_index_list} ${candidate} Candidate ${candidate} not exisiting in ${controller_index_list} - \ Run Keyword If '${candidate}' != '${owner}' Append To List ${candidates_list} ${candidate} - [Return] ${owner} ${candidates_list} - Check Item Occurrence At URI In Cluster [Arguments] ${controller_index_list} ${dictionary_item_occurrence} ${uri} [Documentation] Send a GET with the supplied ${uri} to all cluster instances in ${controller_index_list} diff --git a/csit/libraries/ClusterOpenFlow.robot b/csit/libraries/ClusterOpenFlow.robot index a5cf935997..a3d2560fa0 100644 --- a/csit/libraries/ClusterOpenFlow.robot +++ b/csit/libraries/ClusterOpenFlow.robot @@ -15,20 +15,55 @@ ${operational_port_1} ${OPERATIONAL_NODES_API}/node/openflow:1/node-connector Check OpenFlow Shards Status [Arguments] ${controller_index_list} [Documentation] Check Status for all shards in OpenFlow application. - ${inv_conf_leader} ${inv_conf_followers_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Shard Status ${controller_index_list} - ... config inventory - ${inv_oper_leader} ${inv_oper_followers_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Shard Status ${controller_index_list} - ... operational inventory - ${topo_oper_leader} ${topo_oper_followers_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Shard Status ${controller_index_list} - ... operational topology + ${inv_conf_leader} ${inv_conf_followers_list} Get Cluster Shard Status ${controller_index_list} config inventory + ${inv_oper_leader} ${inv_oper_followers_list} Get Cluster Shard Status ${controller_index_list} operational inventory + ${topo_oper_leader} ${topo_oper_followers_list} Get Cluster Shard Status ${controller_index_list} operational topology + ${owner_oper_leader} ${owner_oper_followers_list} Get Cluster Shard Status ${controller_index_list} operational entity-ownership Log config inventory Leader is ${inv_conf_leader} and followers are ${inv_conf_followers_list} Log operational inventory Leader is ${inv_oper_leader} and followers are ${inv_oper_followers_list} Log operational topology Leader is ${topo_oper_leader} and followers are ${topo_oper_followers_list} + Log operational entity-ownership Leader is ${owner_oper_leader} and followers are ${owner_oper_followers_list} + +Check OpenFlow Shards Status After Cluster Event + [Arguments] ${controller_index_list} + [Documentation] Check Shards Status after some cluster event. + Wait Until Keyword Succeeds 90s 1s Check OpenFlow Shards Status ${controller_index_list} + +Get Cluster Entity Owner For Openflow + [Arguments] ${controller_index_list} ${device_type} ${device} + [Documentation] Checks OpenFlow Entity Owner status for a ${device} and returns owner index and list of candidates from a ${controller_index_list}. + ... ${device_type} is normally openflow. + ${length}= Get Length ${controller_index_list} + ${candidates_list}= Create List + ${data}= Get Data From URI controller@{controller_index_list}[0] /restconf/operational/entity-owners:entity-owners + Log ${data} + ${data}= Replace String ${data} /general-entity:entity[general-entity:name=' ${EMPTY} + ${clear_data}= Replace String ${data} '] ${EMPTY} + Log ${clear_data} + ${json}= To Json ${clear_data} + ${entity_type_list}= Get From Dictionary &{json}[entity-owners] entity-type + ${entity_type_index}= Get Index From List Of Dictionaries ${entity_type_list} type ${device_type} + Should Not Be Equal ${entity_type_index} -1 No Entity Owner found for ${device_type} + ${entity_list}= Get From Dictionary @{entity_type_list}[${entity_type_index}] entity + ${entity_index}= Get Index From List Of Dictionaries ${entity_list} id ${device} + Should Not Be Equal ${entity_index} -1 Device ${device} not found in Entity Owner ${device_type} + ${entity_owner}= Get From Dictionary @{entity_list}[${entity_index}] owner + Should Not Be Empty ${entity_owner} No owner found for ${device} + ${owner}= Replace String ${entity_owner} member- ${EMPTY} + ${owner}= Convert To Integer ${owner} + List Should Contain Value ${controller_index_list} ${owner} Owner ${owner} not exisiting in ${controller_index_list} + ${entity_candidates_list}= Get From Dictionary @{entity_list}[${entity_index}] candidate + ${list_length}= Get Length ${entity_candidates_list} + : FOR ${entity_candidate} IN @{entity_candidates_list} + \ ${candidate}= Replace String &{entity_candidate}[name] member- ${EMPTY} + \ ${candidate}= Convert To Integer ${candidate} + \ Run Keyword If '${candidate}' != '${owner}' Append To List ${candidates_list} ${candidate} + [Return] ${owner} ${candidates_list} Get OpenFlow Entity Owner Status For One Device [Arguments] ${controller_index_list} [Documentation] Check Entity Owner Status and identify owner and candidate. - ${owner} ${candidates_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Entity Owner Status ${controller_index_list} + ${owner} ${candidates_list} Wait Until Keyword Succeeds 10s 1s Get Cluster Entity Owner For Openflow ${controller_index_list} ... openflow openflow:1 [Return] ${owner} ${candidates_list} @@ -53,17 +88,21 @@ Add Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} [Documentation] Add sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}. ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/sample_flow_1.json - ${dictionary}= Create Dictionary 10.0.1.0/24=1 + Run Keyword If '${ODL_OF_PLUGIN}' == 'helium' Set Test Variable &{dictionary} 10.0.1.0/24=2 + Run Keyword If '${ODL_OF_PLUGIN}' == 'lithium' Set Test Variable &{dictionary} 10.0.1.0/24=1 Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + Wait Until Keyword Succeeds 15s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Modify Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} [Documentation] Modify sample flow in ${controller_index} and verify it gets applied in all instances in ${controller_index_list}. ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/sample_flow_2.json - ${dictionary}= Create Dictionary 10.0.2.0/24=1 + ##There are slight differences on the way He and Li plugin display table information. He plugin has an additional Hashmap field + ##replicating some of the matches in the flows section. + Run Keyword If '${ODL_OF_PLUGIN}' == 'helium' Set Test Variable &{dictionary} 10.0.2.0/24=2 + Run Keyword If '${ODL_OF_PLUGIN}' == 'lithium' Set Test Variable &{dictionary} 10.0.2.0/24=1 Put And Check At URI In Cluster ${controller_index_list} ${controller_index} ${config_table_0}/flow/1 ${body} - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + Wait Until Keyword Succeeds 15s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Delete Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} @@ -76,11 +115,12 @@ Send RPC Add Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} [Documentation] Add sample flow in ${controller_index} and verify it gets applied from all instances in ${controller_index_list}. ${body}= OperatingSystem.Get File ${CURDIR}/../variables/openflowplugin/add_flow_rpc.json - ${dictionary}= Create Dictionary 10.0.1.0/24=1 + Run Keyword If '${ODL_OF_PLUGIN}' == 'helium' Set Test Variable &{dictionary} 10.0.1.0/24=2 + Run Keyword If '${ODL_OF_PLUGIN}' == 'lithium' Set Test Variable &{dictionary} 10.0.1.0/24=1 ${resp} RequestsLibrary.Post Request controller${controller_index} /restconf/operations/sal-flow:add-flow ${body} ${HEADERS_YANG_JSON} Log ${resp.content} Should Be Equal As Strings ${resp.status_code} 200 - Wait Until Keyword Succeeds 5s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} + Wait Until Keyword Succeeds 15s 1s Check Item Occurrence At URI In Cluster ${controller_index_list} ${dictionary} ${operational_table_0} Send RPC Delete Sample Flow And Verify [Arguments] ${controller_index_list} ${controller_index} diff --git a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot index ff9e3c970f..3454295405 100644 --- a/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot +++ b/csit/suites/openflowplugin/Clustering/010__Cluster_HA_Owner_Failover.robot @@ -91,14 +91,17 @@ Kill Owner Instance Check Shards Status After Fail [Documentation] Create original cluster list and check Status for all shards in OpenFlow application. - Check OpenFlow Shards Status ${new_cluster_list} + Check OpenFlow Shards Status After Cluster Event ${new_cluster_list} Check Entity Owner Status And Find Owner and Candidate After Fail [Documentation] Check Entity Owner Status and identify owner and candidate. ${new_owner} ${new_candidates_list} Get OpenFlow Entity Owner Status For One Device ${new_cluster_list} + Run Keyword And Continue On Failure List Should Not Contain Value ${new_candidates_list} ${original_owner} Original owner ${original_owner} still in candidate list. + Remove Values From List ${new_candidates_list} ${original_owner} ${new_candidate}= Get From List ${new_candidates_list} 0 Set Suite Variable ${new_owner} Set Suite Variable ${new_candidate} + [Teardown] Report_Failure_Due_To_Bug 5004 Check Network Operational Information After Fail [Documentation] Check device is in operational inventory and topology in all cluster instances. @@ -153,16 +156,16 @@ Restore Network and Verify After Fail Take OpenFlow Device Link Up and Verify ${new_cluster_list} Start Old Owner Instance - [Documentation] Kill Owner Instance and verify it is dead + [Documentation] Start old Owner Instance and verify it is up Start Multiple Controllers 300s ${original_owner} Check Shards Status After Recover [Documentation] Create original cluster list and check Status for all shards in OpenFlow application. - Wait Until Keyword Succeeds 5s 1s Check OpenFlow Shards Status ${original_cluster_list} + Check OpenFlow Shards Status After Cluster Event ${original_cluster_list} Check Entity Owner Status After Recover [Documentation] Check Entity Owner Status and identify owner and candidate. - ${new_owner} ${new_candidates_list} Wait Until Keyword Succeeds 5s 1s Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} + ${new_owner} ${new_candidates_list} Get OpenFlow Entity Owner Status For One Device ${original_cluster_list} Set Suite Variable ${new_owner} Check Network Operational Information After Recover diff --git a/csit/testplans/openflowplugin-clustering-helium-design.txt b/csit/testplans/openflowplugin-clustering-helium-design.txt new file mode 100644 index 0000000000..42ec6a2e1f --- /dev/null +++ b/csit/testplans/openflowplugin-clustering-helium-design.txt @@ -0,0 +1,4 @@ +# Place the suites in run order: +integration/test/csit/suites/openflowplugin/Clustering +integration/test/csit/suites/openflowplugin/Clustering +integration/test/csit/suites/openflowplugin/Sanity3Node diff --git a/csit/variables/Variables.py b/csit/variables/Variables.py index adbf6f92a4..4b01a97a79 100644 --- a/csit/variables/Variables.py +++ b/csit/variables/Variables.py @@ -50,6 +50,9 @@ ODL_NETCONF_PASSWORD = 'admin' ODL_NETCONF_PROMPT = ']]>]]>' ODL_NETCONF_NAMESPACE = 'urn:ietf:params:xml:ns:netconf:base:1.0' +# OpenFlow variables +ODL_OF_PLUGIN = 'lithium' + # VTN Coordinator Variables VTNC = '127.0.0.1' VTNCPORT = '8083' -- 2.36.6