From 1d8068763d89cefb2ac2847a73d67e46506d7f6b Mon Sep 17 00:00:00 2001 From: Tomas Markovic Date: Mon, 13 Nov 2017 09:10:45 +0100 Subject: [PATCH] BMP HA test bmp_ha_karaf_stop Simulate HA with BMP Mock tool Stopping karaf on one node and verifying reconnection. Restarting karaf, verifying its presence. bmp_ha_karaf_restart Killing karaf on one node and verifying reconnection. Restarting karaf, verifying its presence. Reconfiguring timers for HA tests Removed Isolation tests, because right now they are failing. Change-Id: If79b2289850ade1c87864340a2b839c53bc729f3 Signed-off-by: Tomas Markovic --- csit/libraries/ClusterManagement.robot | 17 ++ .../040_bgp_ha_karaf_restart.robot | 19 +-- .../050_bgp_ha_karaf_isolation.robot | 19 +-- .../060_bmp_ha_karaf_restart.robot | 151 ++++++++++++++++++ .../bgpclustering/070_bmp_ha_karaf_stop.robot | 151 ++++++++++++++++++ .../bgpcep-bgpclustering-ha-oxygen.txt | 10 ++ csit/testplans/bgpcep-bgpclustering-ha.txt | 1 - 7 files changed, 349 insertions(+), 19 deletions(-) create mode 100644 csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot create mode 100644 csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot create mode 100644 csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt diff --git a/csit/libraries/ClusterManagement.robot b/csit/libraries/ClusterManagement.robot index 36d73e66d4..f7975d9868 100644 --- a/csit/libraries/ClusterManagement.robot +++ b/csit/libraries/ClusterManagement.robot @@ -50,6 +50,8 @@ ${KARAF_HOME} ${WORKSPACE}${/}${BUNDLEFOLDER} # TODO: Migrate to Variable ${RESTCONF_MODULES_DIR} ${CURDIR}/../variables/restconf/modules ${SINGLETON_NETCONF_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value= ${SINGLETON_NETCONF_DEVICE_ID_SUFFIX} ]]]]}'] +${SINGLETON_BGPCEP_DEVICE_ID_PREFIX} /odl-general-entity:entity[odl-general-entity:name=' +${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} -service-group'] ${SINGLETON_ELECTION_ENTITY_TYPE} org.opendaylight.mdsal.ServiceEntityType ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE} org.opendaylight.mdsal.AsyncServiceCloseEntityType ${NODE_START_COMMAND} ${KARAF_HOME}/bin/start @@ -259,6 +261,21 @@ Get_Owner_And_Candidates_For_Device_Singleton_Netconf BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same [Return] ${owner_1} ${candidate_list_1} +Get_Owner_And_Candidates_For_Device_Singleton_Bgpcep + [Arguments] ${device_name} ${member_index} ${http_timeout}=2 + [Documentation] Returns the owner and a list of candidates for the SB device ${device_name}. Request is sent to member ${member_index}. + # Get election entity type results + ${type} = BuiltIn.Set_Variable ${SINGLETON_ELECTION_ENTITY_TYPE} + ${id} = BuiltIn.Set_Variable ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} + ${owner_1} ${candidate_list_1} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout} + # Get change ownership entity type results + ${type} = BuiltIn.Set_Variable ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE} + ${id} = BuiltIn.Set_Variable ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX} + ${owner_2} ${candidate_list_2} = Get_Owner_And_Candidates_For_Type_And_Id ${type} ${id} ${member_index} http_timeout=${http_timeout} + # Owners must be same, if not, there is still some election or change ownership in progress + BuiltIn.Should_Be_Equal_As_Integers ${owner_1} ${owner_2} Owners for device ${device_name} are not same + [Return] ${owner_1} ${candidate_list_1} + Get_Owner_And_Candidates_For_Device [Arguments] ${device_name} ${device_type} ${member_index} ${http_timeout}=${EMPTY} [Documentation] Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}. diff --git a/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot b/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot index 28f704af7d..2f47f30982 100644 --- a/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot +++ b/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot @@ -37,10 +37,11 @@ ${RIB_INSTANCE} example-bgp-rib *** Test Cases *** Get Example Bgp Rib Owner [Documentation] Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured. - ${rib_owner} ${rib_candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib org.opendaylight.mdsal.ServiceEntityType 1 - BuiltIn.Set Suite variable ${rib_owner} ${rib_owner} - BuiltIn.Set Suite variable ${rib_owner_node_id} ${ODL_SYSTEM_${rib_owner}_IP} - BuiltIn.Set Suite variable ${rib_candidates} ${rib_candidates} + ${rib_owner} ${rib_candidates}= BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Get_Owner_And_Successors_For_Device example-bgp-rib + ... Bgpcep 1 + BuiltIn.Set Suite variable ${rib_owner} + BuiltIn.Log ${ODL_SYSTEM_${rib_owner}_IP} + BuiltIn.Set Suite variable ${rib_candidates} ${session}= Resolve_Http_Session_For_Member member_index=${rib_owner} BuiltIn.Set_Suite_Variable ${living_session} ${session} BuiltIn.Set_Suite_Variable ${living_node} ${rib_owner} @@ -49,7 +50,7 @@ Reconfigure_ODL_To_Accept_Connection [Documentation] Configure BGP peer module with initiate-connection set to false. [Setup] SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing &{mapping} Create Dictionary IP=${TOOLS_SYSTEM_IP} HOLDTIME=${HOLDTIME} PEER_PORT=${BGP_TOOL_PORT} PASSIVE_MODE=true BGP_RIB_OPENCONFIG=${RIB_INSTANCE} - TemplatedRequests.Put_As_Xml_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} + TemplatedRequests.Put_As_Xml_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} http_timeout=5 [Teardown] SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed Start_ExaBgp_Peer @@ -88,7 +89,7 @@ Verify_New_Candidate Verify ExaBgp Still Connected [Documentation] Verifies exabgp's presence in operational ds - Verify_Tools_Connection ${living_session} + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s Verify_Tools_Connection ${living_session} Stop_ExaBgp_Peer [Documentation] Stops exabgp @@ -97,7 +98,7 @@ Stop_ExaBgp_Peer Delete_Bgp_Peer_Configuration [Documentation] Revert the BGP configuration to the original state: without any configured peers &{mapping} Create Dictionary BGP_RIB_OPENCONFIG=${RIB_INSTANCE} IP=${TOOLS_SYSTEM_IP} - TemplatedRequests.Delete_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} + TemplatedRequests.Delete_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} http_timeout=5 *** Keywords *** Setup_Everything @@ -152,13 +153,13 @@ Upload_Config_Files Verify_New_Rib_Owner_Elected [Arguments] ${old_owner} ${node_to_ask} [Documentation] Verifies new owner was elected - ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib org.opendaylight.mdsal.ServiceEntityType ${node_to_ask} + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib Bgpcep ${node_to_ask} BuiltIn.Should_Not_Be_Equal ${old_owner} ${owner} Verify_New_Rib_Candidate_Present [Arguments] ${candidate} ${node_to_ask} [Documentation] Verifies candidate's presence. - ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib org.opendaylight.mdsal.ServiceEntityType ${node_to_ask} + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib Bgpcep ${node_to_ask} BuiltIn.Should_Contain ${candidates} ${candidate} Verify_Tools_Connection diff --git a/csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot b/csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot index 6198fc4592..f7b061b7bc 100644 --- a/csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot +++ b/csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot @@ -37,10 +37,11 @@ ${RIB_INSTANCE} example-bgp-rib *** Test Cases *** Get Example Bgp Rib Owner [Documentation] Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured. - ${rib_owner} ${rib_candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib org.opendaylight.mdsal.ServiceEntityType 1 - BuiltIn.Set Suite variable ${rib_owner} ${rib_owner} - BuiltIn.Set Suite variable ${rib_owner_node_id} ${ODL_SYSTEM_${rib_owner}_IP} - BuiltIn.Set Suite variable ${rib_candidates} ${rib_candidates} + ${rib_owner} ${rib_candidates}= BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Get_Owner_And_Successors_For_Device example-bgp-rib + ... Bgpcep 1 + BuiltIn.Set Suite variable ${rib_owner} + BuiltIn.Log ${ODL_SYSTEM_${rib_owner}_IP} + BuiltIn.Set Suite variable ${rib_candidates} ${session}= Resolve_Http_Session_For_Member member_index=${rib_owner} BuiltIn.Set_Suite_Variable ${living_session} ${session} BuiltIn.Set_Suite_Variable ${living_node} ${rib_owner} @@ -49,7 +50,7 @@ Reconfigure_ODL_To_Accept_Connection [Documentation] Configure BGP peer module with initiate-connection set to false. [Setup] SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing &{mapping} Create Dictionary IP=${TOOLS_SYSTEM_IP} HOLDTIME=${HOLDTIME} PEER_PORT=${BGP_TOOL_PORT} PASSIVE_MODE=true BGP_RIB_OPENCONFIG=${RIB_INSTANCE} - TemplatedRequests.Put_As_Xml_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} + TemplatedRequests.Put_As_Xml_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} http_timeout=5 [Teardown] SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed Start_ExaBgp_Peer @@ -88,7 +89,7 @@ Verify_New_Candidate Verify ExaBgp Still Connected [Documentation] Verifies exabgp's presence in operational ds - Verify_Tools_Connection ${living_session} + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s Verify_Tools_Connection ${living_session} Stop_ExaBgp_Peer [Documentation] Stops exabgp @@ -97,7 +98,7 @@ Stop_ExaBgp_Peer Delete_Bgp_Peer_Configuration [Documentation] Revert the BGP configuration to the original state: without any configured peers &{mapping} Create Dictionary BGP_RIB_OPENCONFIG=${RIB_INSTANCE} IP=${TOOLS_SYSTEM_IP} - TemplatedRequests.Delete_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} + TemplatedRequests.Delete_Templated ${BGP_PEER_FOLDER} mapping=${mapping} session=${living_session} http_timeout=5 *** Keywords *** Setup_Everything @@ -152,13 +153,13 @@ Upload_Config_Files Verify_New_Rib_Owner_Elected [Arguments] ${old_owner} ${node_to_ask} [Documentation] Verifies new owner was elected - ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib org.opendaylight.mdsal.ServiceEntityType ${node_to_ask} + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib Bgpcep ${node_to_ask} BuiltIn.Should_Not_Be_Equal ${old_owner} ${owner} Verify_New_Rib_Candidate_Present [Arguments] ${candidate} ${node_to_ask} [Documentation] Verifies candidate's presence. - ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib org.opendaylight.mdsal.ServiceEntityType ${node_to_ask} + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device example-bgp-rib Bgpcep ${node_to_ask} BuiltIn.Should_Contain ${candidates} ${candidate} Verify_Tools_Connection diff --git a/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot b/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot new file mode 100644 index 0000000000..fe79a45c28 --- /dev/null +++ b/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot @@ -0,0 +1,151 @@ +*** Settings *** +Documentation BMP functional HA testing with BMP mock. +... +... Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... This suite uses BMP mock. It is configured to have 3 peers (all 3 nodes of odl). +... BMP implemented with singleton accepts only one incomming conection. BMP mock +... logs will show that one peer will be connected and two will fail. +... After killing karaf which owned connection new owner should be elected and +... this new owner should accept incomming BMP connection. +Suite Setup Setup_Everything +Suite Teardown Teardown_Everything +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Library SSHLibrary timeout=10s +Library RequestsLibrary +Library Collections +Library OperatingSystem +Resource ../../../variables/Variables.robot +Resource ../../../libraries/SetupUtils.robot +Resource ../../../libraries/ClusterManagement.robot +Resource ../../../libraries/RemoteBash.robot +Resource ../../../libraries/SSHKeywords.robot +Resource ../../../libraries/TemplatedRequests.robot +Resource ../../../libraries/NexusKeywords.robot + +*** Variables *** +${BGP_VAR_FOLDER} ${CURDIR}/../../../variables/bgpclustering +${HOLDTIME} 180 +${BGP_BMP_DIR} ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/filled_structure +${BGP_BMP_FEAT_DIR} ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/empty_structure +${BMP_LOG_FILE} bmpmock.log + +*** Test Cases *** +Get_Example_Bm_Owner + [Documentation] Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured. + ${bm_owner} ${bm_candidates}= Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Get_Owner_And_Successors_For_Device bmp-monitors + ... Bgpcep 1 + BuiltIn.Set Suite variable ${bm_owner} + BuiltIn.Log ${bm_owner} + BuiltIn.Log ${ODL_SYSTEM_${bm_owner}_IP} + BuiltIn.Set Suite variable ${bm_candidates} + ${session}= ClusterManagement.Resolve_Http_Session_For_Member member_index=${bm_owner} + BuiltIn.Set_Suite_Variable ${living_session} ${session} + BuiltIn.Log ${living_session} + BuiltIn.Set_Suite_Variable ${living_node} ${bm_owner} + +Verify_Bmp_Feature + [Documentation] Verify example-bmp-monitor presence in bmp-monitors + &{mapping} BuiltIn.Create_Dictionary TOOL_IP=${TOOLS_SYSTEM_IP} + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s TemplatedRequests.Get_As_Json_Templated folder=${BGP_BMP_FEAT_DIR} mapping=${mapping} verify=True + ... session=${living_session} + +Start_Bmp_Mock + [Documentation] Starts bmp mock + ${command}= NexusKeywords.Compose_Full_Java_Command -jar ${filename} --local_address ${TOOLS_SYSTEM_IP} --remote_address ${ODL_SYSTEM_1_IP}:12345,${ODL_SYSTEM_2_IP}:12345,${ODL_SYSTEM_3_IP}:12345 --routers_count 1 --peers_count 1 --log_level TRACE 2>&1 | tee ${BMP_LOG_FILE} + BuiltIn.Log ${command} + SSHLibrary.Set_Client_Configuration timeout=30s + SSHLibrary.Write ${command} + ${output}= SSHLibrary.Read_Until successfully established. + BuiltIn.Log ${output} + +Verify_Data_Reported_1 + [Documentation] Verifies if example-bmp-monitor reported expected data + Verify_Data_Reported + +Stop_Current_Owner_Member + [Documentation] Killing cluster node which is connected with bmp mock. + ClusterManagement.Kill_Single_Member ${bm_owner} + BuiltIn.Set Suite variable ${old_bm_owner} ${bm_owner} + BuiltIn.Set Suite variable ${old_bm_candidates} ${bm_candidates} + ${idx}= Collections.Get From List ${old_bm_candidates} 0 + ${session}= ClusterManagement.Resolve_Http_Session_For_Member member_index=${idx} + BuiltIn.Set_Suite_Variable ${living_session} ${session} + BuiltIn.Set_Suite_Variable ${living_node} ${idx} + +Verify_New_Bm_Owner + [Documentation] Verifies if new owner of bmp-monitor is elected. + BuiltIn.Wait_Until_Keyword_Succeeds 10x 5s Verify_New_Bm_Owner_Elected ${old_bm_owner} ${living_node} + +Verify_Data_Reported_2 + [Documentation] Verifies if example-bmp-monitor reported expected data + Verify_Data_Reported + +Start_Old_Owner_Member + [Documentation] Start killed node + ClusterManagement.Start_Single_Member ${old_bm_owner} + +Verify_New_Candidate + [Documentation] Verifies started node become candidate for bmp-monitor + BuiltIn.Wait_Until_Keyword_Succeeds 10x 5s Verify_New_Bm_Candidate_Present ${old_bm_owner} ${living_node} + +Verify_Data_Reported_3 + [Documentation] Verifies if example-bmp-monitor reported expected data + Verify_Data_Reported + +Stop_Bmp_Mock + [Documentation] Send ctrl+c to bmp-mock to stop it + RemoteBash.Write_Bare_Ctrl_C + ${output}= SSHLibrary.Read_Until_Prompt + BuiltIn.Log ${output} + +*** Keywords *** +Setup_Everything + [Documentation] Initial setup + SetupUtils.Setup_Utils_For_Setup_And_Teardown + ${odl1} = SSHKeywords.Open_Connection_To_ODL_System ip_address=${ODL_SYSTEM_1_IP} + SSHLibrary.Put_File ${CURDIR}/../../../../tools/deployment/search.sh + SSHLibrary.Close_Connection + ${odl2} = SSHKeywords.Open_Connection_To_ODL_System ip_address=${ODL_SYSTEM_2_IP} + SSHLibrary.Put_File ${CURDIR}/../../../../tools/deployment/search.sh + SSHLibrary.Close_Connection + ${odl3} = SSHKeywords.Open_Connection_To_ODL_System ip_address=${ODL_SYSTEM_3_IP} + SSHLibrary.Put_File ${CURDIR}/../../../../tools/deployment/search.sh + SSHLibrary.Close_Connection + ClusterManagement.ClusterManagement_Setup + SSHKeywords.Open_Connection_To_Tools_System + ${name}= NexusKeywords.Deploy_Test_Tool bgpcep bgp-bmp-mock + BuiltIn.Set_Suite_Variable ${filename} ${name} + +Teardown_Everything + [Documentation] Suite cleanup + SSHLibrary.Get_File ${BMP_LOG_FILE} + ${cnt}= OperatingSystem.Get_File ${BMP_LOG_FILE} + Log ${cnt} + RequestsLibrary.Delete_All_Sessions + SSHLibrary.Close_All_Connections + +Verify_Data_Reported + [Arguments] ${ip}=${TOOLS_SYSTEM_IP} + [Documentation] Verifies if the tool reported expected data + &{mapping} BuiltIn.Create_Dictionary TOOL_IP=${ip} + ${output}= Wait Until Keyword Succeeds 10x 5s TemplatedRequests.Get_As_Json_Templated folder=${BGP_BMP_DIR} mapping=${mapping} + ... session=${living_session} verify=True + Log ${output} + +Verify_New_Bm_Owner_Elected + [Arguments] ${old_owner} ${node_to_ask} + [Documentation] Verifies new owner was elected + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device bmp-monitors Bgpcep ${node_to_ask} + BuiltIn.Should_Not_Be_Equal ${old_owner} ${owner} + +Verify_New_Bm_Candidate_Present + [Arguments] ${candidate} ${node_to_ask} + [Documentation] Verifies candidate's presence. + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device bmp-monitors Bgpcep ${node_to_ask} + BuiltIn.Should_Contain ${candidates} ${candidate} diff --git a/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot b/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot new file mode 100644 index 0000000000..98db3dc189 --- /dev/null +++ b/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot @@ -0,0 +1,151 @@ +*** Settings *** +Documentation BMP functional HA testing with one BMP mock. +... +... Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... This suite uses BMP mock. It is configured to have 3 peers (all 3 nodes of odl). +... BMP implemented with singleton accepts only one incomming conection. BMP mock +... logs will show that one peer will be connected and two will fail. +... After stopping karaf which owned connection new owner should be elected and +... this new owner should accept incomming BMP connection. +Suite Setup Setup_Everything +Suite Teardown Teardown_Everything +Test Setup SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed +Library SSHLibrary timeout=10s +Library RequestsLibrary +Library Collections +Library OperatingSystem +Resource ../../../variables/Variables.robot +Resource ../../../libraries/SetupUtils.robot +Resource ../../../libraries/ClusterManagement.robot +Resource ../../../libraries/RemoteBash.robot +Resource ../../../libraries/SSHKeywords.robot +Resource ../../../libraries/TemplatedRequests.robot +Resource ../../../libraries/NexusKeywords.robot + +*** Variables *** +${BGP_VAR_FOLDER} ${CURDIR}/../../../variables/bgpclustering +${HOLDTIME} 180 +${BGP_BMP_DIR} ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/filled_structure +${BGP_BMP_FEAT_DIR} ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/empty_structure +${BMP_LOG_FILE} bmpmock.log + +*** Test Cases *** +Get_Example_Bm_Owner + [Documentation] Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured. + ${bm_owner} ${bm_candidates}= Wait_Until_Keyword_Succeeds 5x 2s ClusterManagement.Get_Owner_And_Successors_For_Device bmp-monitors + ... Bgpcep 1 + BuiltIn.Set Suite variable ${bm_owner} + BuiltIn.Log ${bm_owner} + BuiltIn.Log ${ODL_SYSTEM_${bm_owner}_IP} + BuiltIn.Set Suite variable ${bm_candidates} + ${session}= ClusterManagement.Resolve_Http_Session_For_Member member_index=${bm_owner} + BuiltIn.Set_Suite_Variable ${living_session} ${session} + BuiltIn.Log ${living_session} + BuiltIn.Set_Suite_Variable ${living_node} ${bm_owner} + +Verify_Bmp_Feature + [Documentation] Verify example-bmp-monitor presence in bmp-monitors + &{mapping} BuiltIn.Create_Dictionary TOOL_IP=${TOOLS_SYSTEM_IP} + BuiltIn.Wait_Until_Keyword_Succeeds 5x 2s TemplatedRequests.Get_As_Json_Templated folder=${BGP_BMP_FEAT_DIR} mapping=${mapping} verify=True + ... session=${living_session} + +Start_Bmp_Mock + [Documentation] Starts bmp mock + ${command}= NexusKeywords.Compose_Full_Java_Command -jar ${filename} --local_address ${TOOLS_SYSTEM_IP} --remote_address ${ODL_SYSTEM_1_IP}:12345,${ODL_SYSTEM_2_IP}:12345,${ODL_SYSTEM_3_IP}:12345 --routers_count 1 --peers_count 1 --log_level TRACE 2>&1 | tee ${BMP_LOG_FILE} + BuiltIn.Log ${command} + SSHLibrary.Set_Client_Configuration timeout=30s + SSHLibrary.Write ${command} + ${output}= SSHLibrary.Read_Until successfully established. + BuiltIn.Log ${output} + +Verify_Data_Reported_1 + [Documentation] Verifies if example-bmp-monitor reported expected data + Verify_Data_Reported + +Stop_Current_Owner_Member + [Documentation] Stop cluster node which is connected with bmp mock. + ClusterManagement.Stop_Single_Member ${bm_owner} + BuiltIn.Set Suite variable ${old_bm_owner} ${bm_owner} + BuiltIn.Set Suite variable ${old_bm_candidates} ${bm_candidates} + ${idx}= Collections.Get From List ${old_bm_candidates} 0 + ${session}= ClusterManagement.Resolve_Http_Session_For_Member member_index=${idx} + BuiltIn.Set_Suite_Variable ${living_session} ${session} + BuiltIn.Set_Suite_Variable ${living_node} ${idx} + +Verify_New_Bm_Owner + [Documentation] Verifies if new owner of bmp-monitor is elected. + BuiltIn.Wait_Until_Keyword_Succeeds 10x 5s Verify_New_Bm_Owner_Elected ${old_bm_owner} ${living_node} + +Verify_Data_Reported_2 + [Documentation] Verifies if example-bmp-monitor reported expected data + Verify_Data_Reported + +Start_Old_Owner_Member + [Documentation] Restart old owner node with bin/start + ClusterManagement.Start_Single_Member ${old_bm_owner} + +Verify_New_Candidate + [Documentation] Verifies started node become candidate for bmp-monitor + BuiltIn.Wait_Until_Keyword_Succeeds 10x 5s Verify_New_Bm_Candidate_Present ${old_bm_owner} ${living_node} + +Verify_Data_Reported_3 + [Documentation] Verifies if example-bmp-monitor reported expected data + Verify_Data_Reported + +Stop_Bmp_Mock + [Documentation] Send ctrl+c to bmp-mock to stop it + RemoteBash.Write_Bare_Ctrl_C + ${output}= SSHLibrary.Read_Until_Prompt + BuiltIn.Log ${output} + +*** Keywords *** +Setup_Everything + [Documentation] Initial setup + SetupUtils.Setup_Utils_For_Setup_And_Teardown + ${odl1} = SSHKeywords.Open_Connection_To_ODL_System ip_address=${ODL_SYSTEM_1_IP} + SSHLibrary.Put_File ${CURDIR}/../../../../tools/deployment/search.sh + SSHLibrary.Close_Connection + ${odl2} = SSHKeywords.Open_Connection_To_ODL_System ip_address=${ODL_SYSTEM_2_IP} + SSHLibrary.Put_File ${CURDIR}/../../../../tools/deployment/search.sh + SSHLibrary.Close_Connection + ${odl3} = SSHKeywords.Open_Connection_To_ODL_System ip_address=${ODL_SYSTEM_3_IP} + SSHLibrary.Put_File ${CURDIR}/../../../../tools/deployment/search.sh + SSHLibrary.Close_Connection + ClusterManagement.ClusterManagement_Setup + SSHKeywords.Open_Connection_To_Tools_System + ${name}= NexusKeywords.Deploy_Test_Tool bgpcep bgp-bmp-mock + BuiltIn.Set_Suite_Variable ${filename} ${name} + +Teardown_Everything + [Documentation] Suite cleanup + SSHLibrary.Get_File ${BMP_LOG_FILE} + ${cnt}= OperatingSystem.Get_File ${BMP_LOG_FILE} + Log ${cnt} + RequestsLibrary.Delete_All_Sessions + SSHLibrary.Close_All_Connections + +Verify_Data_Reported + [Arguments] ${ip}=${TOOLS_SYSTEM_IP} + [Documentation] Verifies if the tool reported expected data + &{mapping} BuiltIn.Create_Dictionary TOOL_IP=${ip} + ${output}= Wait Until Keyword Succeeds 10x 5s TemplatedRequests.Get_As_Json_Templated folder=${BGP_BMP_DIR} mapping=${mapping} + ... session=${living_session} verify=True + Log ${output} + +Verify_New_Bm_Owner_Elected + [Arguments] ${old_owner} ${node_to_ask} + [Documentation] Verifies new owner was elected + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device bmp-monitors Bgpcep ${node_to_ask} + BuiltIn.Should_Not_Be_Equal ${old_owner} ${owner} + +Verify_New_Bm_Candidate_Present + [Arguments] ${candidate} ${node_to_ask} + [Documentation] Verifies candidate's presence. + ${owner} ${candidates}= ClusterManagement.Get_Owner_And_Successors_For_device bmp-monitors Bgpcep ${node_to_ask} + BuiltIn.Should_Contain ${candidates} ${candidate} diff --git a/csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt b/csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt new file mode 100644 index 0000000000..0bd6df9f17 --- /dev/null +++ b/csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt @@ -0,0 +1,10 @@ +# Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License v1.0 which accompanies this distribution, +# and is available at http://www.eclipse.org/legal/epl-v10.html + +# Place the suites in run order: +integration/test/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot +integration/test/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot +integration/test/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot diff --git a/csit/testplans/bgpcep-bgpclustering-ha.txt b/csit/testplans/bgpcep-bgpclustering-ha.txt index ed4f8b3493..b5802b427e 100644 --- a/csit/testplans/bgpcep-bgpclustering-ha.txt +++ b/csit/testplans/bgpcep-bgpclustering-ha.txt @@ -6,4 +6,3 @@ # Place the suites in run order: integration/test/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot -integration/test/csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot -- 2.36.6