BMP HA test 71/65471/170
authorTomas Markovic <tomas.markovic@pantheon.tech>
Mon, 13 Nov 2017 08:10:45 +0000 (09:10 +0100)
committerLuis Gomez <ecelgp@gmail.com>
Mon, 15 Jan 2018 23:44:07 +0000 (23:44 +0000)
bmp_ha_karaf_stop
Simulate HA with BMP Mock tool
Stopping karaf on one node and verifying reconnection.
Restarting karaf, verifying its presence.

bmp_ha_karaf_restart
Killing karaf on one node and verifying reconnection.
Restarting karaf, verifying its presence.

Reconfiguring timers for HA tests

Removed Isolation tests, because right now they are failing.

Change-Id: If79b2289850ade1c87864340a2b839c53bc729f3
Signed-off-by: Tomas Markovic <tomas.markovic@pantheon.tech>
csit/libraries/ClusterManagement.robot
csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot
csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot
csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot [new file with mode: 0644]
csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot [new file with mode: 0644]
csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt [new file with mode: 0644]
csit/testplans/bgpcep-bgpclustering-ha.txt

index 36d73e66d47efa6a54e6120008bc5286db1ff01b..f7975d9868ad3c060d90d4f154bf7d85496e3835 100644 (file)
@@ -50,6 +50,8 @@ ${KARAF_HOME}     ${WORKSPACE}${/}${BUNDLEFOLDER}    # TODO: Migrate to Variable
 ${RESTCONF_MODULES_DIR}    ${CURDIR}/../variables/restconf/modules
 ${SINGLETON_NETCONF_DEVICE_ID_PREFIX}    /odl-general-entity:entity[odl-general-entity:name='KeyedInstanceIdentifier{targetType=interface org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node, path=[org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology, org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology[key=TopologyKey [_topologyId=Uri [_value=topology-netconf]]], org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node[key=NodeKey [_nodeId=Uri [_value=
 ${SINGLETON_NETCONF_DEVICE_ID_SUFFIX}    ]]]]}']
+${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}    /odl-general-entity:entity[odl-general-entity:name='
+${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}    -service-group']
 ${SINGLETON_ELECTION_ENTITY_TYPE}    org.opendaylight.mdsal.ServiceEntityType
 ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}    org.opendaylight.mdsal.AsyncServiceCloseEntityType
 ${NODE_START_COMMAND}    ${KARAF_HOME}/bin/start
@@ -259,6 +261,21 @@ Get_Owner_And_Candidates_For_Device_Singleton_Netconf
     BuiltIn.Should_Be_Equal_As_Integers    ${owner_1}    ${owner_2}    Owners for device ${device_name} are not same
     [Return]    ${owner_1}    ${candidate_list_1}
 
+Get_Owner_And_Candidates_For_Device_Singleton_Bgpcep
+    [Arguments]    ${device_name}    ${member_index}    ${http_timeout}=2
+    [Documentation]    Returns the owner and a list of candidates for the SB device ${device_name}. Request is sent to member ${member_index}.
+    # Get election entity type results
+    ${type} =    BuiltIn.Set_Variable    ${SINGLETON_ELECTION_ENTITY_TYPE}
+    ${id} =    BuiltIn.Set_Variable    ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}
+    ${owner_1}    ${candidate_list_1} =    Get_Owner_And_Candidates_For_Type_And_Id    ${type}    ${id}    ${member_index}    http_timeout=${http_timeout}
+    # Get change ownership entity type results
+    ${type} =    BuiltIn.Set_Variable    ${SINGLETON_CHANGE_OWNERSHIP_ENTITY_TYPE}
+    ${id} =    BuiltIn.Set_Variable    ${SINGLETON_BGPCEP_DEVICE_ID_PREFIX}${device_name}${SINGLETON_BGPCEP_DEVICE_ID_SUFFIX}
+    ${owner_2}    ${candidate_list_2} =    Get_Owner_And_Candidates_For_Type_And_Id    ${type}    ${id}    ${member_index}    http_timeout=${http_timeout}
+    # Owners must be same, if not, there is still some election or change ownership in progress
+    BuiltIn.Should_Be_Equal_As_Integers    ${owner_1}    ${owner_2}    Owners for device ${device_name} are not same
+    [Return]    ${owner_1}    ${candidate_list_1}
+
 Get_Owner_And_Candidates_For_Device
     [Arguments]    ${device_name}    ${device_type}    ${member_index}    ${http_timeout}=${EMPTY}
     [Documentation]    Returns the owner and a list of candidates for the SB device ${device_name} of type ${device_type}. Request is sent to member ${member_index}.
index 28f704af7de4afee1f4add816c301f45edcbd1a4..2f47f309825852c8916139d16732e148bcf58552 100644 (file)
@@ -37,10 +37,11 @@ ${RIB_INSTANCE}    example-bgp-rib
 *** Test Cases ***
 Get Example Bgp Rib Owner
     [Documentation]    Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured.
-    ${rib_owner}    ${rib_candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    org.opendaylight.mdsal.ServiceEntityType    1
-    BuiltIn.Set Suite variable    ${rib_owner}    ${rib_owner}
-    BuiltIn.Set Suite variable    ${rib_owner_node_id}    ${ODL_SYSTEM_${rib_owner}_IP}
-    BuiltIn.Set Suite variable    ${rib_candidates}    ${rib_candidates}
+    ${rib_owner}    ${rib_candidates}=    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Get_Owner_And_Successors_For_Device    example-bgp-rib
+    ...    Bgpcep    1
+    BuiltIn.Set Suite variable    ${rib_owner}
+    BuiltIn.Log    ${ODL_SYSTEM_${rib_owner}_IP}
+    BuiltIn.Set Suite variable    ${rib_candidates}
     ${session}=    Resolve_Http_Session_For_Member    member_index=${rib_owner}
     BuiltIn.Set_Suite_Variable    ${living_session}    ${session}
     BuiltIn.Set_Suite_Variable    ${living_node}    ${rib_owner}
@@ -49,7 +50,7 @@ Reconfigure_ODL_To_Accept_Connection
     [Documentation]    Configure BGP peer module with initiate-connection set to false.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     &{mapping}    Create Dictionary    IP=${TOOLS_SYSTEM_IP}    HOLDTIME=${HOLDTIME}    PEER_PORT=${BGP_TOOL_PORT}    PASSIVE_MODE=true    BGP_RIB_OPENCONFIG=${RIB_INSTANCE}
-    TemplatedRequests.Put_As_Xml_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}
+    TemplatedRequests.Put_As_Xml_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}    http_timeout=5
     [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Start_ExaBgp_Peer
@@ -88,7 +89,7 @@ Verify_New_Candidate
 
 Verify ExaBgp Still Connected
     [Documentation]    Verifies exabgp's presence in operational ds
-    Verify_Tools_Connection    ${living_session}
+    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Verify_Tools_Connection    ${living_session}
 
 Stop_ExaBgp_Peer
     [Documentation]    Stops exabgp
@@ -97,7 +98,7 @@ Stop_ExaBgp_Peer
 Delete_Bgp_Peer_Configuration
     [Documentation]    Revert the BGP configuration to the original state: without any configured peers
     &{mapping}    Create Dictionary    BGP_RIB_OPENCONFIG=${RIB_INSTANCE}    IP=${TOOLS_SYSTEM_IP}
-    TemplatedRequests.Delete_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}
+    TemplatedRequests.Delete_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}    http_timeout=5
 
 *** Keywords ***
 Setup_Everything
@@ -152,13 +153,13 @@ Upload_Config_Files
 Verify_New_Rib_Owner_Elected
     [Arguments]    ${old_owner}    ${node_to_ask}
     [Documentation]    Verifies new owner was elected
-    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    org.opendaylight.mdsal.ServiceEntityType    ${node_to_ask}
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    Bgpcep    ${node_to_ask}
     BuiltIn.Should_Not_Be_Equal    ${old_owner}    ${owner}
 
 Verify_New_Rib_Candidate_Present
     [Arguments]    ${candidate}    ${node_to_ask}
     [Documentation]    Verifies candidate's presence.
-    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    org.opendaylight.mdsal.ServiceEntityType    ${node_to_ask}
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    Bgpcep    ${node_to_ask}
     BuiltIn.Should_Contain    ${candidates}    ${candidate}
 
 Verify_Tools_Connection
index 6198fc4592800c4b81ba039c674227e9f1fe636a..f7b061b7bc3f960f56064e13930dfbaeedf8dece 100644 (file)
@@ -37,10 +37,11 @@ ${RIB_INSTANCE}    example-bgp-rib
 *** Test Cases ***
 Get Example Bgp Rib Owner
     [Documentation]    Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured.
-    ${rib_owner}    ${rib_candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    org.opendaylight.mdsal.ServiceEntityType    1
-    BuiltIn.Set Suite variable    ${rib_owner}    ${rib_owner}
-    BuiltIn.Set Suite variable    ${rib_owner_node_id}    ${ODL_SYSTEM_${rib_owner}_IP}
-    BuiltIn.Set Suite variable    ${rib_candidates}    ${rib_candidates}
+    ${rib_owner}    ${rib_candidates}=    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Get_Owner_And_Successors_For_Device    example-bgp-rib
+    ...    Bgpcep    1
+    BuiltIn.Set Suite variable    ${rib_owner}
+    BuiltIn.Log    ${ODL_SYSTEM_${rib_owner}_IP}
+    BuiltIn.Set Suite variable    ${rib_candidates}
     ${session}=    Resolve_Http_Session_For_Member    member_index=${rib_owner}
     BuiltIn.Set_Suite_Variable    ${living_session}    ${session}
     BuiltIn.Set_Suite_Variable    ${living_node}    ${rib_owner}
@@ -49,7 +50,7 @@ Reconfigure_ODL_To_Accept_Connection
     [Documentation]    Configure BGP peer module with initiate-connection set to false.
     [Setup]    SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
     &{mapping}    Create Dictionary    IP=${TOOLS_SYSTEM_IP}    HOLDTIME=${HOLDTIME}    PEER_PORT=${BGP_TOOL_PORT}    PASSIVE_MODE=true    BGP_RIB_OPENCONFIG=${RIB_INSTANCE}
-    TemplatedRequests.Put_As_Xml_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}
+    TemplatedRequests.Put_As_Xml_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}    http_timeout=5
     [Teardown]    SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
 
 Start_ExaBgp_Peer
@@ -88,7 +89,7 @@ Verify_New_Candidate
 
 Verify ExaBgp Still Connected
     [Documentation]    Verifies exabgp's presence in operational ds
-    Verify_Tools_Connection    ${living_session}
+    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    Verify_Tools_Connection    ${living_session}
 
 Stop_ExaBgp_Peer
     [Documentation]    Stops exabgp
@@ -97,7 +98,7 @@ Stop_ExaBgp_Peer
 Delete_Bgp_Peer_Configuration
     [Documentation]    Revert the BGP configuration to the original state: without any configured peers
     &{mapping}    Create Dictionary    BGP_RIB_OPENCONFIG=${RIB_INSTANCE}    IP=${TOOLS_SYSTEM_IP}
-    TemplatedRequests.Delete_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}
+    TemplatedRequests.Delete_Templated    ${BGP_PEER_FOLDER}    mapping=${mapping}    session=${living_session}    http_timeout=5
 
 *** Keywords ***
 Setup_Everything
@@ -152,13 +153,13 @@ Upload_Config_Files
 Verify_New_Rib_Owner_Elected
     [Arguments]    ${old_owner}    ${node_to_ask}
     [Documentation]    Verifies new owner was elected
-    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    org.opendaylight.mdsal.ServiceEntityType    ${node_to_ask}
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    Bgpcep    ${node_to_ask}
     BuiltIn.Should_Not_Be_Equal    ${old_owner}    ${owner}
 
 Verify_New_Rib_Candidate_Present
     [Arguments]    ${candidate}    ${node_to_ask}
     [Documentation]    Verifies candidate's presence.
-    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    org.opendaylight.mdsal.ServiceEntityType    ${node_to_ask}
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    example-bgp-rib    Bgpcep    ${node_to_ask}
     BuiltIn.Should_Contain    ${candidates}    ${candidate}
 
 Verify_Tools_Connection
diff --git a/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot b/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot
new file mode 100644 (file)
index 0000000..fe79a45
--- /dev/null
@@ -0,0 +1,151 @@
+*** Settings ***
+Documentation     BMP functional HA testing with BMP mock.
+...
+...               Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+...
+...               This program and the accompanying materials are made available under the
+...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
+...               and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+...               This suite uses BMP mock. It is configured to have 3 peers (all 3 nodes of odl).
+...               BMP implemented with singleton accepts only one incomming conection. BMP mock
+...               logs will show that one peer will be connected and two will fail.
+...               After killing karaf which owned connection new owner should be elected and
+...               this new owner should accept incomming BMP connection.
+Suite Setup       Setup_Everything
+Suite Teardown    Teardown_Everything
+Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Library           SSHLibrary    timeout=10s
+Library           RequestsLibrary
+Library           Collections
+Library           OperatingSystem
+Resource          ../../../variables/Variables.robot
+Resource          ../../../libraries/SetupUtils.robot
+Resource          ../../../libraries/ClusterManagement.robot
+Resource          ../../../libraries/RemoteBash.robot
+Resource          ../../../libraries/SSHKeywords.robot
+Resource          ../../../libraries/TemplatedRequests.robot
+Resource          ../../../libraries/NexusKeywords.robot
+
+*** Variables ***
+${BGP_VAR_FOLDER}    ${CURDIR}/../../../variables/bgpclustering
+${HOLDTIME}       180
+${BGP_BMP_DIR}    ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/filled_structure
+${BGP_BMP_FEAT_DIR}    ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/empty_structure
+${BMP_LOG_FILE}    bmpmock.log
+
+*** Test Cases ***
+Get_Example_Bm_Owner
+    [Documentation]    Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured.
+    ${bm_owner}    ${bm_candidates}=    Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Get_Owner_And_Successors_For_Device    bmp-monitors
+    ...    Bgpcep    1
+    BuiltIn.Set Suite variable    ${bm_owner}
+    BuiltIn.Log    ${bm_owner}
+    BuiltIn.Log    ${ODL_SYSTEM_${bm_owner}_IP}
+    BuiltIn.Set Suite variable    ${bm_candidates}
+    ${session}=    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${bm_owner}
+    BuiltIn.Set_Suite_Variable    ${living_session}    ${session}
+    BuiltIn.Log    ${living_session}
+    BuiltIn.Set_Suite_Variable    ${living_node}    ${bm_owner}
+
+Verify_Bmp_Feature
+    [Documentation]    Verify example-bmp-monitor presence in bmp-monitors
+    &{mapping}    BuiltIn.Create_Dictionary    TOOL_IP=${TOOLS_SYSTEM_IP}
+    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    TemplatedRequests.Get_As_Json_Templated    folder=${BGP_BMP_FEAT_DIR}    mapping=${mapping}    verify=True
+    ...    session=${living_session}
+
+Start_Bmp_Mock
+    [Documentation]    Starts bmp mock
+    ${command}=    NexusKeywords.Compose_Full_Java_Command    -jar ${filename} --local_address ${TOOLS_SYSTEM_IP} --remote_address ${ODL_SYSTEM_1_IP}:12345,${ODL_SYSTEM_2_IP}:12345,${ODL_SYSTEM_3_IP}:12345 --routers_count 1 --peers_count 1 --log_level TRACE 2>&1 | tee ${BMP_LOG_FILE}
+    BuiltIn.Log    ${command}
+    SSHLibrary.Set_Client_Configuration    timeout=30s
+    SSHLibrary.Write    ${command}
+    ${output}=    SSHLibrary.Read_Until    successfully established.
+    BuiltIn.Log    ${output}
+
+Verify_Data_Reported_1
+    [Documentation]    Verifies if example-bmp-monitor reported expected data
+    Verify_Data_Reported
+
+Stop_Current_Owner_Member
+    [Documentation]    Killing cluster node which is connected with bmp mock.
+    ClusterManagement.Kill_Single_Member    ${bm_owner}
+    BuiltIn.Set Suite variable    ${old_bm_owner}    ${bm_owner}
+    BuiltIn.Set Suite variable    ${old_bm_candidates}    ${bm_candidates}
+    ${idx}=    Collections.Get From List    ${old_bm_candidates}    0
+    ${session}=    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${idx}
+    BuiltIn.Set_Suite_Variable    ${living_session}    ${session}
+    BuiltIn.Set_Suite_Variable    ${living_node}    ${idx}
+
+Verify_New_Bm_Owner
+    [Documentation]    Verifies if new owner of bmp-monitor is elected.
+    BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Verify_New_Bm_Owner_Elected    ${old_bm_owner}    ${living_node}
+
+Verify_Data_Reported_2
+    [Documentation]    Verifies if example-bmp-monitor reported expected data
+    Verify_Data_Reported
+
+Start_Old_Owner_Member
+    [Documentation]    Start killed node
+    ClusterManagement.Start_Single_Member    ${old_bm_owner}
+
+Verify_New_Candidate
+    [Documentation]    Verifies started node become candidate for bmp-monitor
+    BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Verify_New_Bm_Candidate_Present    ${old_bm_owner}    ${living_node}
+
+Verify_Data_Reported_3
+    [Documentation]    Verifies if example-bmp-monitor reported expected data
+    Verify_Data_Reported
+
+Stop_Bmp_Mock
+    [Documentation]    Send ctrl+c to bmp-mock to stop it
+    RemoteBash.Write_Bare_Ctrl_C
+    ${output}=    SSHLibrary.Read_Until_Prompt
+    BuiltIn.Log    ${output}
+
+*** Keywords ***
+Setup_Everything
+    [Documentation]    Initial setup
+    SetupUtils.Setup_Utils_For_Setup_And_Teardown
+    ${odl1} =    SSHKeywords.Open_Connection_To_ODL_System    ip_address=${ODL_SYSTEM_1_IP}
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/deployment/search.sh
+    SSHLibrary.Close_Connection
+    ${odl2} =    SSHKeywords.Open_Connection_To_ODL_System    ip_address=${ODL_SYSTEM_2_IP}
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/deployment/search.sh
+    SSHLibrary.Close_Connection
+    ${odl3} =    SSHKeywords.Open_Connection_To_ODL_System    ip_address=${ODL_SYSTEM_3_IP}
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/deployment/search.sh
+    SSHLibrary.Close_Connection
+    ClusterManagement.ClusterManagement_Setup
+    SSHKeywords.Open_Connection_To_Tools_System
+    ${name}=    NexusKeywords.Deploy_Test_Tool    bgpcep    bgp-bmp-mock
+    BuiltIn.Set_Suite_Variable    ${filename}    ${name}
+
+Teardown_Everything
+    [Documentation]    Suite cleanup
+    SSHLibrary.Get_File    ${BMP_LOG_FILE}
+    ${cnt}=    OperatingSystem.Get_File    ${BMP_LOG_FILE}
+    Log    ${cnt}
+    RequestsLibrary.Delete_All_Sessions
+    SSHLibrary.Close_All_Connections
+
+Verify_Data_Reported
+    [Arguments]    ${ip}=${TOOLS_SYSTEM_IP}
+    [Documentation]    Verifies if the tool reported expected data
+    &{mapping}    BuiltIn.Create_Dictionary    TOOL_IP=${ip}
+    ${output}=    Wait Until Keyword Succeeds    10x    5s    TemplatedRequests.Get_As_Json_Templated    folder=${BGP_BMP_DIR}    mapping=${mapping}
+    ...    session=${living_session}    verify=True
+    Log    ${output}
+
+Verify_New_Bm_Owner_Elected
+    [Arguments]    ${old_owner}    ${node_to_ask}
+    [Documentation]    Verifies new owner was elected
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    bmp-monitors    Bgpcep    ${node_to_ask}
+    BuiltIn.Should_Not_Be_Equal    ${old_owner}    ${owner}
+
+Verify_New_Bm_Candidate_Present
+    [Arguments]    ${candidate}    ${node_to_ask}
+    [Documentation]    Verifies candidate's presence.
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    bmp-monitors    Bgpcep    ${node_to_ask}
+    BuiltIn.Should_Contain    ${candidates}    ${candidate}
diff --git a/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot b/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot
new file mode 100644 (file)
index 0000000..98db3dc
--- /dev/null
@@ -0,0 +1,151 @@
+*** Settings ***
+Documentation     BMP functional HA testing with one BMP mock.
+...
+...               Copyright (c) 2017 AT&T Intellectual Property. All rights reserved.
+...
+...               This program and the accompanying materials are made available under the
+...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
+...               and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+...               This suite uses BMP mock. It is configured to have 3 peers (all 3 nodes of odl).
+...               BMP implemented with singleton accepts only one incomming conection. BMP mock
+...               logs will show that one peer will be connected and two will fail.
+...               After stopping karaf which owned connection new owner should be elected and
+...               this new owner should accept incomming BMP connection.
+Suite Setup       Setup_Everything
+Suite Teardown    Teardown_Everything
+Test Setup        SetupUtils.Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown     SetupUtils.Teardown_Test_Show_Bugs_If_Test_Failed
+Library           SSHLibrary    timeout=10s
+Library           RequestsLibrary
+Library           Collections
+Library           OperatingSystem
+Resource          ../../../variables/Variables.robot
+Resource          ../../../libraries/SetupUtils.robot
+Resource          ../../../libraries/ClusterManagement.robot
+Resource          ../../../libraries/RemoteBash.robot
+Resource          ../../../libraries/SSHKeywords.robot
+Resource          ../../../libraries/TemplatedRequests.robot
+Resource          ../../../libraries/NexusKeywords.robot
+
+*** Variables ***
+${BGP_VAR_FOLDER}    ${CURDIR}/../../../variables/bgpclustering
+${HOLDTIME}       180
+${BGP_BMP_DIR}    ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/filled_structure
+${BGP_BMP_FEAT_DIR}    ${CURDIR}/../../../variables/bgpfunctional/bmp_basic/empty_structure
+${BMP_LOG_FILE}    bmpmock.log
+
+*** Test Cases ***
+Get_Example_Bm_Owner
+    [Documentation]    Find an odl node which is able to accept incomming connection. To this node netconf connector should be configured.
+    ${bm_owner}    ${bm_candidates}=    Wait_Until_Keyword_Succeeds    5x    2s    ClusterManagement.Get_Owner_And_Successors_For_Device    bmp-monitors
+    ...    Bgpcep    1
+    BuiltIn.Set Suite variable    ${bm_owner}
+    BuiltIn.Log    ${bm_owner}
+    BuiltIn.Log    ${ODL_SYSTEM_${bm_owner}_IP}
+    BuiltIn.Set Suite variable    ${bm_candidates}
+    ${session}=    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${bm_owner}
+    BuiltIn.Set_Suite_Variable    ${living_session}    ${session}
+    BuiltIn.Log    ${living_session}
+    BuiltIn.Set_Suite_Variable    ${living_node}    ${bm_owner}
+
+Verify_Bmp_Feature
+    [Documentation]    Verify example-bmp-monitor presence in bmp-monitors
+    &{mapping}    BuiltIn.Create_Dictionary    TOOL_IP=${TOOLS_SYSTEM_IP}
+    BuiltIn.Wait_Until_Keyword_Succeeds    5x    2s    TemplatedRequests.Get_As_Json_Templated    folder=${BGP_BMP_FEAT_DIR}    mapping=${mapping}    verify=True
+    ...    session=${living_session}
+
+Start_Bmp_Mock
+    [Documentation]    Starts bmp mock
+    ${command}=    NexusKeywords.Compose_Full_Java_Command    -jar ${filename} --local_address ${TOOLS_SYSTEM_IP} --remote_address ${ODL_SYSTEM_1_IP}:12345,${ODL_SYSTEM_2_IP}:12345,${ODL_SYSTEM_3_IP}:12345 --routers_count 1 --peers_count 1 --log_level TRACE 2>&1 | tee ${BMP_LOG_FILE}
+    BuiltIn.Log    ${command}
+    SSHLibrary.Set_Client_Configuration    timeout=30s
+    SSHLibrary.Write    ${command}
+    ${output}=    SSHLibrary.Read_Until    successfully established.
+    BuiltIn.Log    ${output}
+
+Verify_Data_Reported_1
+    [Documentation]    Verifies if example-bmp-monitor reported expected data
+    Verify_Data_Reported
+
+Stop_Current_Owner_Member
+    [Documentation]    Stop cluster node which is connected with bmp mock.
+    ClusterManagement.Stop_Single_Member    ${bm_owner}
+    BuiltIn.Set Suite variable    ${old_bm_owner}    ${bm_owner}
+    BuiltIn.Set Suite variable    ${old_bm_candidates}    ${bm_candidates}
+    ${idx}=    Collections.Get From List    ${old_bm_candidates}    0
+    ${session}=    ClusterManagement.Resolve_Http_Session_For_Member    member_index=${idx}
+    BuiltIn.Set_Suite_Variable    ${living_session}    ${session}
+    BuiltIn.Set_Suite_Variable    ${living_node}    ${idx}
+
+Verify_New_Bm_Owner
+    [Documentation]    Verifies if new owner of bmp-monitor is elected.
+    BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Verify_New_Bm_Owner_Elected    ${old_bm_owner}    ${living_node}
+
+Verify_Data_Reported_2
+    [Documentation]    Verifies if example-bmp-monitor reported expected data
+    Verify_Data_Reported
+
+Start_Old_Owner_Member
+    [Documentation]    Restart old owner node with bin/start
+    ClusterManagement.Start_Single_Member    ${old_bm_owner}
+
+Verify_New_Candidate
+    [Documentation]    Verifies started node become candidate for bmp-monitor
+    BuiltIn.Wait_Until_Keyword_Succeeds    10x    5s    Verify_New_Bm_Candidate_Present    ${old_bm_owner}    ${living_node}
+
+Verify_Data_Reported_3
+    [Documentation]    Verifies if example-bmp-monitor reported expected data
+    Verify_Data_Reported
+
+Stop_Bmp_Mock
+    [Documentation]    Send ctrl+c to bmp-mock to stop it
+    RemoteBash.Write_Bare_Ctrl_C
+    ${output}=    SSHLibrary.Read_Until_Prompt
+    BuiltIn.Log    ${output}
+
+*** Keywords ***
+Setup_Everything
+    [Documentation]    Initial setup
+    SetupUtils.Setup_Utils_For_Setup_And_Teardown
+    ${odl1} =    SSHKeywords.Open_Connection_To_ODL_System    ip_address=${ODL_SYSTEM_1_IP}
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/deployment/search.sh
+    SSHLibrary.Close_Connection
+    ${odl2} =    SSHKeywords.Open_Connection_To_ODL_System    ip_address=${ODL_SYSTEM_2_IP}
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/deployment/search.sh
+    SSHLibrary.Close_Connection
+    ${odl3} =    SSHKeywords.Open_Connection_To_ODL_System    ip_address=${ODL_SYSTEM_3_IP}
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/deployment/search.sh
+    SSHLibrary.Close_Connection
+    ClusterManagement.ClusterManagement_Setup
+    SSHKeywords.Open_Connection_To_Tools_System
+    ${name}=    NexusKeywords.Deploy_Test_Tool    bgpcep    bgp-bmp-mock
+    BuiltIn.Set_Suite_Variable    ${filename}    ${name}
+
+Teardown_Everything
+    [Documentation]    Suite cleanup
+    SSHLibrary.Get_File    ${BMP_LOG_FILE}
+    ${cnt}=    OperatingSystem.Get_File    ${BMP_LOG_FILE}
+    Log    ${cnt}
+    RequestsLibrary.Delete_All_Sessions
+    SSHLibrary.Close_All_Connections
+
+Verify_Data_Reported
+    [Arguments]    ${ip}=${TOOLS_SYSTEM_IP}
+    [Documentation]    Verifies if the tool reported expected data
+    &{mapping}    BuiltIn.Create_Dictionary    TOOL_IP=${ip}
+    ${output}=    Wait Until Keyword Succeeds    10x    5s    TemplatedRequests.Get_As_Json_Templated    folder=${BGP_BMP_DIR}    mapping=${mapping}
+    ...    session=${living_session}    verify=True
+    Log    ${output}
+
+Verify_New_Bm_Owner_Elected
+    [Arguments]    ${old_owner}    ${node_to_ask}
+    [Documentation]    Verifies new owner was elected
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    bmp-monitors    Bgpcep    ${node_to_ask}
+    BuiltIn.Should_Not_Be_Equal    ${old_owner}    ${owner}
+
+Verify_New_Bm_Candidate_Present
+    [Arguments]    ${candidate}    ${node_to_ask}
+    [Documentation]    Verifies candidate's presence.
+    ${owner}    ${candidates}=    ClusterManagement.Get_Owner_And_Successors_For_device    bmp-monitors    Bgpcep    ${node_to_ask}
+    BuiltIn.Should_Contain    ${candidates}    ${candidate}
diff --git a/csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt b/csit/testplans/bgpcep-bgpclustering-ha-oxygen.txt
new file mode 100644 (file)
index 0000000..0bd6df9
--- /dev/null
@@ -0,0 +1,10 @@
+# Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License v1.0 which accompanies this distribution,
+# and is available at http://www.eclipse.org/legal/epl-v10.html
+
+# Place the suites in run order:
+integration/test/csit/suites/bgpcep/bgpclustering/060_bmp_ha_karaf_restart.robot
+integration/test/csit/suites/bgpcep/bgpclustering/070_bmp_ha_karaf_stop.robot
+integration/test/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot
index ed4f8b349382208036d236ea179aaf661d7a7fc4..b5802b427ea7761137119ef54d4bfa5bb8b57cf9 100644 (file)
@@ -6,4 +6,3 @@
 
 # Place the suites in run order:
 integration/test/csit/suites/bgpcep/bgpclustering/040_bgp_ha_karaf_restart.robot
-integration/test/csit/suites/bgpcep/bgpclustering/050_bgp_ha_karaf_isolation.robot