From: Radovan Sajben Date: Thu, 26 Nov 2015 15:45:27 +0000 (+0100) Subject: BGP application peer performance suite X-Git-Tag: release/beryllium-sr1~251 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=integration%2Ftest.git;a=commitdiff_plain;h=aa1e4fd12128953ac1b5549a14fb296d0e27ef6b BGP application peer performance suite - prefill 100k routes via restconf single POST request - add 100k routes via POST requests (one route per request) - check IPv4-topology changes and updates towards BGP peer - check updates towards reconnected BGP peer - check routes withdrawal towards BG peer after routes deletion Change-Id: I40d8c8076064eac710b791c65877dd62cb1e760b Signed-off-by: Radovan Sajben --- diff --git a/csit/libraries/KarafKeywords.robot b/csit/libraries/KarafKeywords.robot index 303213b9c2..fb0771c65e 100644 --- a/csit/libraries/KarafKeywords.robot +++ b/csit/libraries/KarafKeywords.robot @@ -14,8 +14,8 @@ Check Karaf Log File Does Not Have Messages [Arguments] ${ip} ${message} ${user}=${CONTROLLER_USER} ${password}=${CONTROLLER_PASSWORD} ${prompt}=${DEFAULT_LINUX_PROMPT} ${log_file}=${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log [Documentation] Fails if the provided ${message} is found in the karaf.log file. Uses grep to search. The ... karaf.log file can be overridden with ${log_file} to be any file on the given system @ ${ip} - ${output}= Run Command On Controller ${ip} grep ${message} ${log_file} user=${user} password=${password} prompt=${prompt} - Should Not Contain ${output} ${message} + ${output}= Run Command On Controller ${ip} grep -c ${message} ${log_file} user=${user} password=${password} prompt=${prompt} + Should_Be_Equal_As_Strings ${output} 0 Verify Feature Is Installed [Arguments] ${feature_name} ${controller}=${CONTROLLER} ${karaf_port}=${KARAF_SHELL_PORT} diff --git a/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot b/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot new file mode 100644 index 0000000000..891a20bb72 --- /dev/null +++ b/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot @@ -0,0 +1,272 @@ +*** Settings *** +Documentation BGP performance of ingesting from 1 BGP application peer +... +... Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. +... +... This program and the accompanying materials are made available under the +... terms of the Eclipse Public License v1.0 which accompanies this distribution, +... and is available at http://www.eclipse.org/legal/epl-v10.html +... +... Test suite performs basic BGP performance test cases for +... BGP application peer. BGP application peer introduces routes - +... using restconf - in two steps: +... 1. introduces the ${PREFILL} number of routes in one POST request +... 2. POSTs the rest of routes (up to the ${COUNT} number) one by one +... Test suite checks that the prefixes are propagated to +... IPv4 topology and announced to BGP peer via updates. Test case +... where the BGP peer is disconnected and reconnected and all routes +... are deleted by BGP application peer are performed as well. +... Brief description how to configure BGP application peer and +... how to use restconf application peer interface: +... https://wiki.opendaylight.org/view/BGP_LS_PCEP:User_Guide#BGP_Application_Peer +... https://wiki.opendaylight.org/view/BGP_LS_PCEP:Programmer_Guide#BGP +... +... Reported bugs: +... Bug 4689 - Not a reasonable duration of 1M prefix introduction from BGP application peer via restconf +... Bug 4791 - BGPSessionImpl: Failed to send message Update logged even all UPDATE mesages received by iBGP peer +Suite Setup Setup_Everything +Suite Teardown Teardown_Everything +Test Setup Setup_Test_With_Logging_And_Without_Fast_Failing +Test Teardown FailFast.Start_Failing_Fast_If_This_Failed +Force Tags critical +Library SSHLibrary timeout=10s +Library RequestsLibrary +Variables ${CURDIR}/../../../variables/Variables.py +Resource ${CURDIR}/../../../libraries/BGPSpeaker.robot +Resource ${CURDIR}/../../../libraries/ConfigViaRestconf.robot +Resource ${CURDIR}/../../../libraries/FailFast.robot +Resource ${CURDIR}/../../../libraries/KillPythonTool.robot +Resource ${CURDIR}/../../../libraries/PrefixCounting.robot +Resource ${CURDIR}/../../../libraries/SetupUtils.robot + +*** Variables *** +${BGP_VARIABLES_FOLDER} ${CURDIR}/../../../variables/bgpuser/ +${HOLDTIME} 180 +${HOLDTIME_APP_PEER_PREFIX_COUNT} ${HOLDTIME} +${COUNT} 200000 +${PREFILL} 100000 +${COUNT_APP_PEER_PREFIX_COUNT} ${COUNT} +${CHECK_PERIOD} 1 +${CHECK_PERIOD_APP_PEER_PREFIX_COUNT} ${CHECK_PERIOD} +${REPETITIONS_APP_PEER_PREFIX_COUNT} 1 +${BGP_PEER_LOG_LEVEL} info +${BGP_APP_PEER_LOG_LEVEL} info +${CONTROLLER_LOG_LEVEL} INFO +${CONTROLLER_BGP_LOG_LEVEL} DEFAULT +${BGP_PEER_COMMAND} python play.py --amount 0 --myip=${TOOLS_SYSTEM_IP} --myport=${BGP_TOOL_PORT} --peerip=${ODL_SYSTEM_IP} --peerport=${ODL_BGP_PORT} --${BGP_PEER_LOG_LEVEL} +${BGP_PEER_OPTIONS} &>bgp_peer.log +${BGP_APP_PEER_ID} 10.0.0.10 +${BGP_APP_PEER_INITIAL_COMMAND} python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command post --count ${PREFILL} --prefix 8.0.0.0 --prefixlen 28 --${BGP_APP_PEER_LOG_LEVEL} +${BGP_APP_PEER_PUT_COMMAND} python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command put --count ${PREFILL} --prefix 8.0.0.0 --prefixlen 28 --${BGP_APP_PEER_LOG_LEVEL} +${BGP_APP_PEER_DELETE_ALL_COMMAND} python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command delete-all --${BGP_APP_PEER_LOG_LEVEL} +${BGP_APP_PEER_GET_COMMAND} python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command get --${BGP_APP_PEER_LOG_LEVEL} +${BGP_APP_PEER_OPTIONS} &>bgp_app_peer.log +${TEST_DURATION_MULTIPLIER} 30 +${last_prefix_count} -1 + +*** Test Cases *** +Check_For_Empty_Ipv4_Topology_Before_Starting + [Documentation] Wait for example-ipv4-topology to come up and empty. Give large timeout for case when BGP boots slower than restconf. + BuiltIn.Wait_Until_Keyword_Succeeds 120s 1s PrefixCounting.Check_Ipv4_Topology_Is_Empty + +Reconfigure_ODL_To_Accept_Connection + [Documentation] Configure BGP peer module with initiate-connection set to false. + ${template_as_string}= BuiltIn.Set_Variable {'NAME': 'example-bgp-peer', 'IP': '${TOOLS_SYSTEM_IP}', 'HOLDTIME': '${HOLDTIME_APP_PEER_PREFIX_COUNT}', 'PEER_PORT': '${BGP_TOOL_PORT}', 'INITIATE': 'false'} + ConfigViaRestconf.Put_Xml_Template_Folder_Config_Via_Restconf ${BGP_VARIABLES_FOLDER}${/}bgp_peer ${template_as_string} + +Reconfigure_ODL_To_Accept_BGP_Application_Peer + [Documentation] Configure BGP application peer module. + ${template_as_string}= BuiltIn.Set_Variable {'NAME': 'example-bgp-peer-app', 'IP': '${BGP_APP_PEER_ID}'} + ConfigViaRestconf.Put_Xml_Template_Folder_Config_Via_Restconf ${BGP_VARIABLES_FOLDER}${/}bgp_application_peer ${template_as_string} + +Connect_BGP_Peer + [Documentation] Start BGP peer tool + SSHLibrary.Switch Connection bgp_peer_console + Start_Console_Tool ${BGP_PEER_COMMAND} ${BGP_PEER_OPTIONS} + Read_And_Fail_If_Prompt_Is_Seen + +BGP_Application_Peer_Prefill_Routes + [Documentation] Start BGP application peer tool and prefill routes. + SSHLibrary.Switch Connection bgp_app_peer_console + Start_Console_Tool ${BGP_APP_PEER_INITIAL_COMMAND} ${BGP_APP_PEER_OPTIONS} + Wait_Until_Console_Tool_Finish ${bgp_filling_timeout} + Store_File_To_Workspace bgp_app_peer.log bgp_app_peer_prefill.log + +Wait_For_Ipv4_Topology_Is_Prefilled + [Documentation] Wait until example-ipv4-topology reaches the target prfix count. + BuiltIn.Wait Until Keyword Succeeds ${bgp_filling_timeout} 10s PrefixCounting.Check_Ipv4_Topology_Count ${PREFILL} + [Teardown] Report_Failure_Due_To_Bug 4689 + +Check_Bgp_Peer_Updates_For_Prefilled_Routes + [Documentation] Count the routes introduced by updates. + [Setup] SetupUtils.Setup_Test_With_Logging_And_Fast_Failing + BuiltIn.Wait Until Keyword Succeeds ${bgp_filling_timeout} 1s Check_File_For_Word_Count bgp_peer.log total_received_nlri_prefix_counter: ${PREFILL} 2 + [Teardown] Report_Failure_Due_To_Bug 4689 + +BGP_Application_Peer_Introduce_Single_Routes + [Documentation] Start BGP application peer tool and introduce routes. + SSHLibrary.Switch Connection bgp_app_peer_console + Start_Console_Tool python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command add --count ${remaining_prefixes} --prefix 12.0.0.0 --prefixlen 28 --${BGP_APP_PEER_LOG_LEVEL} ${BGP_APP_PEER_OPTIONS} + Wait_Until_Console_Tool_Finish ${bgp_filling_timeout} + Store_File_To_Workspace bgp_app_peer.log bgp_app_peer_singles.log + +Wait_For_Ipv4_Topology_Is_Filled + [Documentation] Wait until example-ipv4-topology reaches the target prfix count. + BuiltIn.Wait Until Keyword Succeeds ${bgp_filling_timeout} 10s PrefixCounting.Check_Ipv4_Topology_Count ${COUNT_APP_PEER_PREFIX_COUNT} + [Teardown] Report_Failure_Due_To_Bug 4689 + +Check_Bgp_Peer_Updates_For_All_Routes + [Documentation] Count the routes introduced by updates. + [Setup] SetupUtils.Setup_Test_With_Logging_And_Fast_Failing + BuiltIn.Wait Until Keyword Succeeds ${bgp_filling_timeout} 1s Check_File_For_Word_Count bgp_peer.log total_received_nlri_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT} 2 + [Teardown] Report_Failure_Due_To_Bug 4689 + +Disconnect_BGP_Peer + [Documentation] Stop BGP peer tool + SSHLibrary.Switch Connection bgp_peer_console + Stop_Console_Tool + Store_File_To_Workspace bgp_peer.log bgp_peer_reconnect.log + +Reconnect_BGP_Peer + [Documentation] Start BGP peer tool + SSHLibrary.Switch Connection bgp_peer_console + Start_Console_Tool ${BGP_PEER_COMMAND} ${BGP_PEER_OPTIONS} + Read_And_Fail_If_Prompt_Is_Seen + +Check_Bgp_Peer_Updates_For_Reintroduced_Routes + [Documentation] Count the routes introduced by updates. + BuiltIn.Wait Until Keyword Succeeds ${bgp_filling_timeout} 1s Check_File_For_Word_Count bgp_peer.log total_received_nlri_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT} 2 + +BGP_Application_Peer_Delete_All_Routes + [Documentation] Start BGP application peer tool and delete all routes. + SSHLibrary.Switch Connection bgp_app_peer_console + Start_Console_Tool ${BGP_APP_PEER_DELETE_ALL_COMMAND} ${BGP_APP_PEER_OPTIONS} + Wait_Until_Console_Tool_Finish ${bgp_emptying_timeout} + Store_File_To_Workspace bgp_app_peer.log bgp_app_peer_delete_all.log + +Wait_For_Stable_Topology_After_Deletion + [Documentation] Wait until example-ipv4-topology becomes stable again. + PrefixCounting.Wait_For_Ipv4_Topology_Prefixes_To_Become_Stable timeout=${bgp_emptying_timeout} period=${CHECK_PERIOD_APP_PEER_PREFIX_COUNT} repetitions=${REPETITIONS_APP_PEER_PREFIX_COUNT} excluded_count=${COUNT_APP_PEER_PREFIX_COUNT} + +Check_For_Empty_Ipv4_Topology_After_Deleting + [Documentation] Example-ipv4-topology should be empty now. + PrefixCounting.Check_Ipv4_Topology_Is_Empty + +Check_Bgp_Peer_Updates_For_Prefix_Withdrawals + [Documentation] Count the routes withdrawn by updates. + [Setup] SetupUtils.Setup_Test_With_Logging_And_Fast_Failing + BuiltIn.Wait Until Keyword Succeeds ${bgp_emptying_timeout} 1s Check_File_For_Word_Count bgp_peer.log total_received_withdrawn_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT} 2 + [Teardown] Report_Failure_Due_To_Bug 4689 + +Stop_BGP_Peer + [Documentation] Stop BGP peer tool + SSHLibrary.Switch Connection bgp_peer_console + Stop_Console_Tool + Store_File_To_Workspace bgp_peer.log bgp_peer_reconnect.log + +Delete_Bgp_Peer_Configuration + [Documentation] Revert the BGP configuration to the original state: without any configured peers. + ${template_as_string}= BuiltIn.Set_Variable {'NAME': 'example-bgp-peer'} + ConfigViaRestconf.Delete_Xml_Template_Folder_Config_Via_Restconf ${BGP_VARIABLES_FOLDER}${/}bgp_peer ${template_as_string} + +Delete_Bgp_Application_Peer_Configuration + [Documentation] Revert the BGP configuration to the original state: without any configured peers. + ${template_as_string}= BuiltIn.Set_Variable {'NAME': 'example-bgp-peer-app'} + ConfigViaRestconf.Delete_Xml_Template_Folder_Config_Via_Restconf ${BGP_VARIABLES_FOLDER}${/}bgp_application_peer ${template_as_string} + +Check_Controller_Logs + [Documentation] Check controller's log for errors + Check Karaf Log File Does Not Have Messages ${ODL_SYSTEM_IP} Failed to send message Update + [Teardown] Report_Failure_Due_To_Bug 4791 + +*** Keywords *** +Setup_Everything + [Documentation] Setup imported resources, SSH-login to tools system, + ... create HTTP session, put Python tool to tools system. + SetupUtils.Setup_Utils_For_Setup_And_Teardown + ConfigViaRestconf.Setup_Config_Via_Restconf + PrefixCounting.PC_Setup + SSHLibrary.Set_Default_Configuration prompt=${TOOLS_SYSTEM_PROMPT} + RequestsLibrary.Create_Session operational http://${ODL_SYSTEM_IP}:${RESTCONFPORT}${OPERATIONAL_API} auth=${AUTH} + # TODO: Do not include slash in ${OPERATIONAL_TOPO_API}, having it typed here is more readable. + # TODO: Alternatively, create variable in Variables which starts with http. + # Both TODOs would probably need to update every suite relying on current Variables. + Open_BGP_Peer_Console + Open_BGP_Aplicationp_Peer_Console + SSHLibrary.Put_File ${CURDIR}/../../../../tools/fastbgp/play.py + SSHLibrary.Put_File ${CURDIR}/../../../../tools/fastbgp/bgp_app_peer.py + SSHLibrary.Put_File ${CURDIR}/../../../../tools/fastbgp/ipv4-routes-template.xml + # Calculate the timeout value based on how many routes are going to be pushed. + # The offset (20) is set for keeping reasonable timeout for low COUNT values. + ${timeout} = BuiltIn.Evaluate ${TEST_DURATION_MULTIPLIER} * ${COUNT_APP_PEER_PREFIX_COUNT} * 3.0 / 10000 + 20 + Builtin.Set_Suite_Variable ${bgp_filling_timeout} ${timeout} + Builtin.Set_Suite_Variable ${bgp_emptying_timeout} ${bgp_filling_timeout*3.0/4} + ${result} = BuiltIn.Evaluate str(int(${COUNT_APP_PEER_PREFIX_COUNT}) - int(${PREFILL})) + Builtin.Set_Suite_Variable ${remaining_prefixes} ${result} + KarafKeywords.Execute_Controller_Karaf_Command_On_Background log:set ${CONTROLLER_LOG_LEVEL} + KarafKeywords.Execute_Controller_Karaf_Command_On_Background log:set ${CONTROLLER_BGP_LOG_LEVEL} org.opendaylight.bgpcep + KarafKeywords.Execute_Controller_Karaf_Command_On_Background log:set ${CONTROLLER_BGP_LOG_LEVEL} org.opendaylight.protocol + +Teardown_Everything + [Documentation] Make sure Python tool was killed and tear down imported Resources. + SSHLibrary.Switch Connection bgp_peer_console + KillPythonTool.Search_And_Kill_Remote_Python 'play\.py' + KillPythonTool.Search_And_Kill_Remote_Python 'bgp_app_peer\.py' + ConfigViaRestconf.Teardown_Config_Via_Restconf + RequestsLibrary.Delete_All_Sessions + SSHLibrary.Close_All_Connections + +Open_BGP_Peer_Console + [Documentation] Create a session for BGP peer. + SSHLibrary.Open_Connection ${TOOLS_SYSTEM_IP} alias=bgp_peer_console + Utils.Flexible_Mininet_Login + +Open_BGP_Aplicationp_Peer_Console + [Documentation] Create a session for BGP peer. + SSHLibrary.Open_Connection ${TOOLS_SYSTEM_IP} alias=bgp_app_peer_console + Utils.Flexible_Mininet_Login + +Start_Console_Tool + [Arguments] ${command} ${tool_opt} + [Documentation] Start the tool ${command} ${tool_opt} + BuiltIn.Log ${command} + ${output}= SSHLibrary.Write ${command} ${tool_opt} + BuiltIn.Log ${output} + +Wait_Until_Console_Tool_Finish + [Arguments] ${timeout} + [Documentation] Wait ${timeout} for the tool exit. + BuiltIn.Wait Until Keyword Succeeds ${timeout} 1s SSHLibrary.Read Until Prompt + +Stop_Console_Tool + [Documentation] Stop the tool if still running. + Utils.Write_Bare_Ctrl_C + ${output}= SSHLibrary.Read delay=1s + BuiltIn.Log ${output} + +Read_And_Fail_If_Prompt_Is_Seen + [Documentation] Try to read SSH to see prompt, but expect to see no prompt within SSHLibrary's timeout. + ${passed}= BuiltIn.Run_Keyword_And_Return_Status BuiltIn.Run_Keyword_And_Expect_Error No match found for '${TOOLS_SYSTEM_PROMPT}' in *. Read_Text_Before_Prompt + BuiltIn.Return_From_Keyword_If ${passed} + BGPSpeaker.Dump_BGP_Speaker_Logs + Builtin.Fail The prompt was seen but it was not expected yet + +Read_Text_Before_Prompt + [Documentation] Log text gathered by SSHLibrary.Read_Until_Prompt. + ... This needs to be a separate keyword just because how Read_And_Fail_If_Prompt_Is_Seen is implemented. + ${text}= SSHLibrary.Read_Until_Prompt + BuiltIn.Log ${text} + +Store_File_To_Workspace + [Arguments] ${source_file_name} ${target_file_name} + [Documentation] Store the ${source_file_name} to the workspace as ${target_file_name}. + ${output_log}= SSHLibrary.Execute_Command cat ${source_file_name} + BuiltIn.Log ${output_log} + Create File ${target_file_name} ${output_log} + +Check_File_For_Word_Count + [Arguments] ${file_name} ${word} ${expected_count} + [Documentation] Count ${word} in ${file_name}. Expect ${expected_count} occurence(s) + ${output_log}= SSHLibrary.Execute_Command grep -o '${word}' ${file_name} | wc -l + BuiltIn.Log ${output_log} + BuiltIn.Should_Be_Equal_As_Strings ${output_log} ${expected_count} diff --git a/csit/testplans/bgpcep-bgp-ingest.txt b/csit/testplans/bgpcep-bgp-ingest.txt index 6de34c9c51..2a7fbcca41 100644 --- a/csit/testplans/bgpcep-bgp-ingest.txt +++ b/csit/testplans/bgpcep-bgp-ingest.txt @@ -8,5 +8,6 @@ integration/test/csit/suites/netconf/ready/netconfready.robot integration/test/csit/suites/bgpcep/bgpingest/singlepeer_prefixcount.robot integration/test/csit/suites/bgpcep/bgpingest/singlepeer_changecount.robot +integration/test/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot integration/test/csit/suites/bgpcep/bgpingest/manypeers_prefixcount.robot integration/test/csit/suites/bgpcep/bgpingest/manypeers_changecount.robot diff --git a/tools/fastbgp/bgp_app_peer.py b/tools/fastbgp/bgp_app_peer.py index aa292a94d1..2af032abd6 100755 --- a/tools/fastbgp/bgp_app_peer.py +++ b/tools/fastbgp/bgp_app_peer.py @@ -37,8 +37,9 @@ def _build_url(odl_ip, port, uri): return url -def _build_data(xml_template, prefix_base, prefix_len, count, element="ipv4-routes"): - """Generate list of routes based on xml templates. +def _stream_data(xml_template, prefix_base, prefix_len, count, element="ipv4-routes"): + """Stream list of routes based on xml template. Memory non-consumable + data generation (on the fly). Args: :xml_template: xml template for routes @@ -52,41 +53,47 @@ def _build_data(xml_template, prefix_base, prefix_len, count, element="ipv4-rout :element: element to be returned Returns: - :returns xml_data: requested element as xml data + :yield xml_data: requested data by elements as xml data """ global total_build_data_time_counter - build_data_timestamp = time.time() routes = md.parse(xml_template) routes_node = routes.getElementsByTagName("ipv4-routes")[0] route_node = routes.getElementsByTagName("ipv4-route")[0] + routes_node.removeChild(route_node) + route_prefix = route_node.getElementsByTagName("prefix")[0] + prefix_gap = 2 ** (32 - prefix_len) + prefix_index_list = range(count) if element == routes_node.tagName: - routes_node.removeChild(route_node) - if count: - prefix_gap = 2 ** (32 - prefix_len) - - for prefix_index in range(count): - new_route_node = route_node.cloneNode(True) - new_route_prefix = new_route_node.getElementsByTagName("prefix")[0] - - prefix = prefix_base + prefix_index * prefix_gap - new_route_prefix.childNodes[0].nodeValue = str(prefix) + "/" + str(prefix_len) - - routes_node.appendChild(new_route_node) - - xml_data = routes_node.toxml() + lines = routes_node.toxml().splitlines() + xml_head = lines[0] + "\n" + xml_tail = "\n".join(lines[1:]) elif element == route_node.tagName: - route_node.setAttribute("xmlns", routes_node.namespaceURI) - route_prefix = route_node.getElementsByTagName("prefix")[0] - route_prefix.childNodes[0].nodeValue = str(prefix_base) + "/" + str(prefix_len) - xml_data = route_node.toxml() + xml_head = "" + xml_tail = "" + route_node.setAttribute("xmlns", route_node.namespaceURI) else: - xml_data = "" - routes.unlink() - logger.debug("xml data generated:\n%s", xml_data) - total_build_data_time_counter += time.time() - build_data_timestamp - return xml_data + prefix_index_list = range(0) + + for prefix_index in prefix_index_list: + build_data_timestamp = time.time() + prefix = prefix_base + prefix_index * prefix_gap + prefix_str = str(prefix) + "/" + str(prefix_len) + route_prefix.childNodes[0].nodeValue = prefix_str + xml_data = route_node.toxml() + if prefix_index == 0: + xml_data = xml_head + xml_data + if prefix_index == len(prefix_index_list) - 1: + xml_data = xml_data + xml_tail + chunk = prefix_index + 1 + if not (chunk % 1000): + logger.info("... streaming chunk %s (prefix: %s)", chunk, prefix_str) + else: + logger.debug("...streaming chunk %s (prefix: %s)", chunk, prefix_str) + logger.debug("xml data\n%s", xml_data) + total_build_data_time_counter += time.time() - build_data_timestamp + yield xml_data def send_request(operation, odl_ip, port, uri, auth, xml_data=None, expect_status_code=200): @@ -125,20 +132,14 @@ def send_request(operation, odl_ip, port, uri, auth, xml_data=None, expect_statu except requests.exceptions.Timeout: logger.error("No response from %s", odl_ip) else: - logger.debug("%s %s", rsp.request, rsp.request.url) - logger.debug("Request headers: %s:", rsp.request.headers) - logger.debug("Request body: %s", rsp.request.body) - logger.debug("Response: %s", rsp.text) if rsp.status_code == expect_status_code: logger.debug("%s %s", rsp.request, rsp.request.url) logger.debug("Request headers: %s:", rsp.request.headers) - logger.debug("Request body: %s", rsp.request.body) logger.debug("Response: %s", rsp.text) logger.debug("%s %s", rsp, rsp.reason) else: logger.error("%s %s", rsp.request, rsp.request.url) logger.error("Request headers: %s:", rsp.request.headers) - logger.error("Request body: %s", rsp.request.body) logger.error("Response: %s", rsp.text) logger.error("%s %s", rsp, rsp.reason) return rsp @@ -214,8 +215,8 @@ def post_prefixes(odl_ip, port, uri, auth, prefix_base=None, prefix_len=None, """ logger.info("Post %s prefix(es) in a single request (starting from %s/%s) into %s:%s/restconf/%s", count, prefix_base, prefix_len, odl_ip, port, uri) - xml_data = _build_data(xml_template, prefix_base, prefix_len, count) - send_request("POST", odl_ip, port, uri, auth, xml_data=xml_data, expect_status_code=204) + xml_stream = _stream_data(xml_template, prefix_base, prefix_len, count) + send_request("POST", odl_ip, port, uri, auth, xml_data=xml_stream, expect_status_code=204) def put_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count, @@ -245,8 +246,8 @@ def put_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count, uri_add_prefix = uri + _uri_suffix_ipv4_routes logger.info("Put %s prefix(es) in a single request (starting from %s/%s) into %s:%s/restconf/%s", count, prefix_base, prefix_len, odl_ip, port, uri_add_prefix) - xml_data = _build_data(xml_template, prefix_base, prefix_len, count) - send_request("PUT", odl_ip, port, uri_add_prefix, auth, xml_data=xml_data) + xml_stream = _stream_data(xml_template, prefix_base, prefix_len, count) + send_request("PUT", odl_ip, port, uri_add_prefix, auth, xml_data=xml_stream) def add_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count, @@ -281,9 +282,9 @@ def add_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count, prefix = prefix_base + prefix_index * prefix_gap logger.info("Adding prefix %s/%s to %s:%s/restconf/%s", prefix, prefix_len, odl_ip, port, uri) - xml_data = _build_data(xml_template, prefix, prefix_len, 1, "ipv4-route") + xml_stream = _stream_data(xml_template, prefix, prefix_len, 1, "ipv4-route") send_request("POST", odl_ip, port, uri_add_prefix, auth, - xml_data=xml_data, expect_status_code=204) + xml_data=xml_stream, expect_status_code=204) def delete_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count, diff --git a/tools/fastbgp/play.py b/tools/fastbgp/play.py index 304bb8f1f5..dcec923def 100755 --- a/tools/fastbgp/play.py +++ b/tools/fastbgp/play.py @@ -1086,6 +1086,7 @@ class ReadTracker(object): self.prefixes_introduced = 0 self.prefixes_withdrawn = 0 self.rx_idle_time = 0 + self.rx_activity_detected = True def read_message_chunk(self): """Read up to one message @@ -1341,20 +1342,27 @@ class ReadTracker(object): wait_timedelta = 0 # And wait for event or something to read. - logger.info("total_received_update_message_counter: %s", - self.updates_received) - logger.info("total_received_nlri_prefix_counter: %s", - self.prefixes_introduced) - logger.info("total_received_withdrawn_prefix_counter: %s", - self.prefixes_withdrawn) + if not self.rx_activity_detected or not (self.updates_received % 100): + # right time to write statistics to the log (not for every update and + # not too frequently to avoid having large log files) + logger.info("total_received_update_message_counter: %s", + self.updates_received) + logger.info("total_received_nlri_prefix_counter: %s", + self.prefixes_introduced) + logger.info("total_received_withdrawn_prefix_counter: %s", + self.prefixes_withdrawn) start_time = time.time() select.select([self.socket], [], [self.socket], wait_timedelta) timedelta = time.time() - start_time self.rx_idle_time += timedelta + self.rx_activity_detected = timedelta < 1 - logger.info("... idle for %.3fs", timedelta) - logger.info("total_rx_idle_time_counter: %.3fs", self.rx_idle_time) + if not self.rx_activity_detected or not (self.updates_received % 100): + # right time to write statistics to the log (not for every update and + # not too frequently to avoid having large log files) + logger.info("... idle for %.3fs", timedelta) + logger.info("total_rx_idle_time_counter: %.3fs", self.rx_idle_time) return