BGP application peer performance suite 56/30256/36
authorRadovan Sajben <rsajben@cisco.com>
Thu, 26 Nov 2015 15:45:27 +0000 (16:45 +0100)
committerGerrit Code Review <gerrit@opendaylight.org>
Sat, 19 Dec 2015 00:11:57 +0000 (00:11 +0000)
- prefill 100k routes via restconf single POST request
- add 100k routes via POST requests (one route per request)
- check IPv4-topology changes and updates towards BGP peer
- check updates towards reconnected BGP peer
- check routes withdrawal towards BG peer after routes deletion

Change-Id: I40d8c8076064eac710b791c65877dd62cb1e760b
Signed-off-by: Radovan Sajben <rsajben@cisco.com>
csit/libraries/KarafKeywords.robot
csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot [new file with mode: 0644]
csit/testplans/bgpcep-bgp-ingest.txt
tools/fastbgp/bgp_app_peer.py
tools/fastbgp/play.py

index 303213b9c28d3cea127cf09a2b5e3d24cfae5069..fb0771c65e52a72531a04041f6c88e277e62cd87 100644 (file)
@@ -14,8 +14,8 @@ Check Karaf Log File Does Not Have Messages
     [Arguments]    ${ip}    ${message}    ${user}=${CONTROLLER_USER}    ${password}=${CONTROLLER_PASSWORD}    ${prompt}=${DEFAULT_LINUX_PROMPT}    ${log_file}=${WORKSPACE}/${BUNDLEFOLDER}/data/log/karaf.log
     [Documentation]    Fails if the provided ${message} is found in the karaf.log file. Uses grep to search. The
     ...    karaf.log file can be overridden with ${log_file} to be any file on the given system @ ${ip}
-    ${output}=    Run Command On Controller    ${ip}    grep ${message} ${log_file}    user=${user}    password=${password}    prompt=${prompt}
-    Should Not Contain    ${output}    ${message}
+    ${output}=    Run Command On Controller    ${ip}    grep -c ${message} ${log_file}    user=${user}    password=${password}    prompt=${prompt}
+    Should_Be_Equal_As_Strings    ${output}    0
 
 Verify Feature Is Installed
     [Arguments]    ${feature_name}    ${controller}=${CONTROLLER}    ${karaf_port}=${KARAF_SHELL_PORT}
diff --git a/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot b/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot
new file mode 100644 (file)
index 0000000..891a20b
--- /dev/null
@@ -0,0 +1,272 @@
+*** Settings ***
+Documentation     BGP performance of ingesting from 1 BGP application peer
+...
+...               Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
+...
+...               This program and the accompanying materials are made available under the
+...               terms of the Eclipse Public License v1.0 which accompanies this distribution,
+...               and is available at http://www.eclipse.org/legal/epl-v10.html
+...
+...               Test suite performs basic BGP performance test cases for
+...               BGP application peer. BGP application peer introduces routes -
+...               using restconf - in two steps:
+...               1. introduces the ${PREFILL} number of routes in one POST request
+...               2. POSTs the rest of routes (up to the ${COUNT} number) one by one
+...               Test suite checks that the prefixes are propagated to
+...               IPv4 topology and announced to BGP peer via updates. Test case
+...               where the BGP peer is disconnected and reconnected and all routes
+...               are deleted by BGP application peer are performed as well.
+...               Brief description how to configure BGP application peer and
+...               how to use restconf application peer interface:
+...               https://wiki.opendaylight.org/view/BGP_LS_PCEP:User_Guide#BGP_Application_Peer
+...               https://wiki.opendaylight.org/view/BGP_LS_PCEP:Programmer_Guide#BGP
+...
+...               Reported bugs:
+...               Bug 4689 - Not a reasonable duration of 1M prefix introduction from BGP application peer via restconf
+...               Bug 4791 - BGPSessionImpl: Failed to send message Update logged even all UPDATE mesages received by iBGP peer
+Suite Setup       Setup_Everything
+Suite Teardown    Teardown_Everything
+Test Setup        Setup_Test_With_Logging_And_Without_Fast_Failing
+Test Teardown     FailFast.Start_Failing_Fast_If_This_Failed
+Force Tags        critical
+Library           SSHLibrary    timeout=10s
+Library           RequestsLibrary
+Variables         ${CURDIR}/../../../variables/Variables.py
+Resource          ${CURDIR}/../../../libraries/BGPSpeaker.robot
+Resource          ${CURDIR}/../../../libraries/ConfigViaRestconf.robot
+Resource          ${CURDIR}/../../../libraries/FailFast.robot
+Resource          ${CURDIR}/../../../libraries/KillPythonTool.robot
+Resource          ${CURDIR}/../../../libraries/PrefixCounting.robot
+Resource          ${CURDIR}/../../../libraries/SetupUtils.robot
+
+*** Variables ***
+${BGP_VARIABLES_FOLDER}    ${CURDIR}/../../../variables/bgpuser/
+${HOLDTIME}       180
+${HOLDTIME_APP_PEER_PREFIX_COUNT}    ${HOLDTIME}
+${COUNT}          200000
+${PREFILL}        100000
+${COUNT_APP_PEER_PREFIX_COUNT}    ${COUNT}
+${CHECK_PERIOD}    1
+${CHECK_PERIOD_APP_PEER_PREFIX_COUNT}    ${CHECK_PERIOD}
+${REPETITIONS_APP_PEER_PREFIX_COUNT}    1
+${BGP_PEER_LOG_LEVEL}    info
+${BGP_APP_PEER_LOG_LEVEL}    info
+${CONTROLLER_LOG_LEVEL}    INFO
+${CONTROLLER_BGP_LOG_LEVEL}    DEFAULT
+${BGP_PEER_COMMAND}    python play.py --amount 0 --myip=${TOOLS_SYSTEM_IP} --myport=${BGP_TOOL_PORT} --peerip=${ODL_SYSTEM_IP} --peerport=${ODL_BGP_PORT} --${BGP_PEER_LOG_LEVEL}
+${BGP_PEER_OPTIONS}    &>bgp_peer.log
+${BGP_APP_PEER_ID}    10.0.0.10
+${BGP_APP_PEER_INITIAL_COMMAND}    python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command post --count ${PREFILL} --prefix 8.0.0.0 --prefixlen 28 --${BGP_APP_PEER_LOG_LEVEL}
+${BGP_APP_PEER_PUT_COMMAND}    python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command put --count ${PREFILL} --prefix 8.0.0.0 --prefixlen 28 --${BGP_APP_PEER_LOG_LEVEL}
+${BGP_APP_PEER_DELETE_ALL_COMMAND}    python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command delete-all --${BGP_APP_PEER_LOG_LEVEL}
+${BGP_APP_PEER_GET_COMMAND}    python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command get --${BGP_APP_PEER_LOG_LEVEL}
+${BGP_APP_PEER_OPTIONS}    &>bgp_app_peer.log
+${TEST_DURATION_MULTIPLIER}    30
+${last_prefix_count}    -1
+
+*** Test Cases ***
+Check_For_Empty_Ipv4_Topology_Before_Starting
+    [Documentation]    Wait for example-ipv4-topology to come up and empty. Give large timeout for case when BGP boots slower than restconf.
+    BuiltIn.Wait_Until_Keyword_Succeeds    120s    1s    PrefixCounting.Check_Ipv4_Topology_Is_Empty
+
+Reconfigure_ODL_To_Accept_Connection
+    [Documentation]    Configure BGP peer module with initiate-connection set to false.
+    ${template_as_string}=    BuiltIn.Set_Variable    {'NAME': 'example-bgp-peer', 'IP': '${TOOLS_SYSTEM_IP}', 'HOLDTIME': '${HOLDTIME_APP_PEER_PREFIX_COUNT}', 'PEER_PORT': '${BGP_TOOL_PORT}', 'INITIATE': 'false'}
+    ConfigViaRestconf.Put_Xml_Template_Folder_Config_Via_Restconf    ${BGP_VARIABLES_FOLDER}${/}bgp_peer    ${template_as_string}
+
+Reconfigure_ODL_To_Accept_BGP_Application_Peer
+    [Documentation]    Configure BGP application peer module.
+    ${template_as_string}=    BuiltIn.Set_Variable    {'NAME': 'example-bgp-peer-app', 'IP': '${BGP_APP_PEER_ID}'}
+    ConfigViaRestconf.Put_Xml_Template_Folder_Config_Via_Restconf    ${BGP_VARIABLES_FOLDER}${/}bgp_application_peer    ${template_as_string}
+
+Connect_BGP_Peer
+    [Documentation]    Start BGP peer tool
+    SSHLibrary.Switch Connection    bgp_peer_console
+    Start_Console_Tool    ${BGP_PEER_COMMAND}    ${BGP_PEER_OPTIONS}
+    Read_And_Fail_If_Prompt_Is_Seen
+
+BGP_Application_Peer_Prefill_Routes
+    [Documentation]    Start BGP application peer tool and prefill routes.
+    SSHLibrary.Switch Connection    bgp_app_peer_console
+    Start_Console_Tool    ${BGP_APP_PEER_INITIAL_COMMAND}    ${BGP_APP_PEER_OPTIONS}
+    Wait_Until_Console_Tool_Finish    ${bgp_filling_timeout}
+    Store_File_To_Workspace    bgp_app_peer.log    bgp_app_peer_prefill.log
+
+Wait_For_Ipv4_Topology_Is_Prefilled
+    [Documentation]    Wait until example-ipv4-topology reaches the target prfix count.
+    BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    10s    PrefixCounting.Check_Ipv4_Topology_Count    ${PREFILL}
+    [Teardown]    Report_Failure_Due_To_Bug    4689
+
+Check_Bgp_Peer_Updates_For_Prefilled_Routes
+    [Documentation]    Count the routes introduced by updates.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
+    BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_nlri_prefix_counter: ${PREFILL}    2
+    [Teardown]    Report_Failure_Due_To_Bug    4689
+
+BGP_Application_Peer_Introduce_Single_Routes
+    [Documentation]    Start BGP application peer tool and introduce routes.
+    SSHLibrary.Switch Connection    bgp_app_peer_console
+    Start_Console_Tool    python bgp_app_peer.py --host ${ODL_SYSTEM_IP} --port ${RESTCONFPORT} --command add --count ${remaining_prefixes} --prefix 12.0.0.0 --prefixlen 28 --${BGP_APP_PEER_LOG_LEVEL}    ${BGP_APP_PEER_OPTIONS}
+    Wait_Until_Console_Tool_Finish    ${bgp_filling_timeout}
+    Store_File_To_Workspace    bgp_app_peer.log    bgp_app_peer_singles.log
+
+Wait_For_Ipv4_Topology_Is_Filled
+    [Documentation]    Wait until example-ipv4-topology reaches the target prfix count.
+    BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    10s    PrefixCounting.Check_Ipv4_Topology_Count    ${COUNT_APP_PEER_PREFIX_COUNT}
+    [Teardown]    Report_Failure_Due_To_Bug    4689
+
+Check_Bgp_Peer_Updates_For_All_Routes
+    [Documentation]    Count the routes introduced by updates.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
+    BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_nlri_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT}    2
+    [Teardown]    Report_Failure_Due_To_Bug    4689
+
+Disconnect_BGP_Peer
+    [Documentation]    Stop BGP peer tool
+    SSHLibrary.Switch Connection    bgp_peer_console
+    Stop_Console_Tool
+    Store_File_To_Workspace    bgp_peer.log    bgp_peer_reconnect.log
+
+Reconnect_BGP_Peer
+    [Documentation]    Start BGP peer tool
+    SSHLibrary.Switch Connection    bgp_peer_console
+    Start_Console_Tool    ${BGP_PEER_COMMAND}    ${BGP_PEER_OPTIONS}
+    Read_And_Fail_If_Prompt_Is_Seen
+
+Check_Bgp_Peer_Updates_For_Reintroduced_Routes
+    [Documentation]    Count the routes introduced by updates.
+    BuiltIn.Wait Until Keyword Succeeds    ${bgp_filling_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_nlri_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT}    2
+
+BGP_Application_Peer_Delete_All_Routes
+    [Documentation]    Start BGP application peer tool and delete all routes.
+    SSHLibrary.Switch Connection    bgp_app_peer_console
+    Start_Console_Tool    ${BGP_APP_PEER_DELETE_ALL_COMMAND}    ${BGP_APP_PEER_OPTIONS}
+    Wait_Until_Console_Tool_Finish    ${bgp_emptying_timeout}
+    Store_File_To_Workspace    bgp_app_peer.log    bgp_app_peer_delete_all.log
+
+Wait_For_Stable_Topology_After_Deletion
+    [Documentation]    Wait until example-ipv4-topology becomes stable again.
+    PrefixCounting.Wait_For_Ipv4_Topology_Prefixes_To_Become_Stable    timeout=${bgp_emptying_timeout}    period=${CHECK_PERIOD_APP_PEER_PREFIX_COUNT}    repetitions=${REPETITIONS_APP_PEER_PREFIX_COUNT}    excluded_count=${COUNT_APP_PEER_PREFIX_COUNT}
+
+Check_For_Empty_Ipv4_Topology_After_Deleting
+    [Documentation]    Example-ipv4-topology should be empty now.
+    PrefixCounting.Check_Ipv4_Topology_Is_Empty
+
+Check_Bgp_Peer_Updates_For_Prefix_Withdrawals
+    [Documentation]    Count the routes withdrawn by updates.
+    [Setup]    SetupUtils.Setup_Test_With_Logging_And_Fast_Failing
+    BuiltIn.Wait Until Keyword Succeeds    ${bgp_emptying_timeout}    1s    Check_File_For_Word_Count    bgp_peer.log    total_received_withdrawn_prefix_counter: ${COUNT_APP_PEER_PREFIX_COUNT}    2
+    [Teardown]    Report_Failure_Due_To_Bug    4689
+
+Stop_BGP_Peer
+    [Documentation]    Stop BGP peer tool
+    SSHLibrary.Switch Connection    bgp_peer_console
+    Stop_Console_Tool
+    Store_File_To_Workspace    bgp_peer.log    bgp_peer_reconnect.log
+
+Delete_Bgp_Peer_Configuration
+    [Documentation]    Revert the BGP configuration to the original state: without any configured peers.
+    ${template_as_string}=    BuiltIn.Set_Variable    {'NAME': 'example-bgp-peer'}
+    ConfigViaRestconf.Delete_Xml_Template_Folder_Config_Via_Restconf    ${BGP_VARIABLES_FOLDER}${/}bgp_peer    ${template_as_string}
+
+Delete_Bgp_Application_Peer_Configuration
+    [Documentation]    Revert the BGP configuration to the original state: without any configured peers.
+    ${template_as_string}=    BuiltIn.Set_Variable    {'NAME': 'example-bgp-peer-app'}
+    ConfigViaRestconf.Delete_Xml_Template_Folder_Config_Via_Restconf    ${BGP_VARIABLES_FOLDER}${/}bgp_application_peer    ${template_as_string}
+
+Check_Controller_Logs
+    [Documentation]    Check controller's log for errors
+    Check Karaf Log File Does Not Have Messages    ${ODL_SYSTEM_IP}    Failed to send message Update
+    [Teardown]    Report_Failure_Due_To_Bug    4791
+
+*** Keywords ***
+Setup_Everything
+    [Documentation]    Setup imported resources, SSH-login to tools system,
+    ...    create HTTP session, put Python tool to tools system.
+    SetupUtils.Setup_Utils_For_Setup_And_Teardown
+    ConfigViaRestconf.Setup_Config_Via_Restconf
+    PrefixCounting.PC_Setup
+    SSHLibrary.Set_Default_Configuration    prompt=${TOOLS_SYSTEM_PROMPT}
+    RequestsLibrary.Create_Session    operational    http://${ODL_SYSTEM_IP}:${RESTCONFPORT}${OPERATIONAL_API}    auth=${AUTH}
+    # TODO: Do not include slash in ${OPERATIONAL_TOPO_API}, having it typed here is more readable.
+    # TODO: Alternatively, create variable in Variables which starts with http.
+    # Both TODOs would probably need to update every suite relying on current Variables.
+    Open_BGP_Peer_Console
+    Open_BGP_Aplicationp_Peer_Console
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/fastbgp/play.py
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/fastbgp/bgp_app_peer.py
+    SSHLibrary.Put_File    ${CURDIR}/../../../../tools/fastbgp/ipv4-routes-template.xml
+    # Calculate the timeout value based on how many routes are going to be pushed.
+    # The offset (20) is set for keeping reasonable timeout for low COUNT values.
+    ${timeout} =    BuiltIn.Evaluate    ${TEST_DURATION_MULTIPLIER} * ${COUNT_APP_PEER_PREFIX_COUNT} * 3.0 / 10000 + 20
+    Builtin.Set_Suite_Variable    ${bgp_filling_timeout}    ${timeout}
+    Builtin.Set_Suite_Variable    ${bgp_emptying_timeout}    ${bgp_filling_timeout*3.0/4}
+    ${result} =    BuiltIn.Evaluate    str(int(${COUNT_APP_PEER_PREFIX_COUNT}) - int(${PREFILL}))
+    Builtin.Set_Suite_Variable    ${remaining_prefixes}    ${result}
+    KarafKeywords.Execute_Controller_Karaf_Command_On_Background    log:set ${CONTROLLER_LOG_LEVEL}
+    KarafKeywords.Execute_Controller_Karaf_Command_On_Background    log:set ${CONTROLLER_BGP_LOG_LEVEL} org.opendaylight.bgpcep
+    KarafKeywords.Execute_Controller_Karaf_Command_On_Background    log:set ${CONTROLLER_BGP_LOG_LEVEL} org.opendaylight.protocol
+
+Teardown_Everything
+    [Documentation]    Make sure Python tool was killed and tear down imported Resources.
+    SSHLibrary.Switch Connection    bgp_peer_console
+    KillPythonTool.Search_And_Kill_Remote_Python    'play\.py'
+    KillPythonTool.Search_And_Kill_Remote_Python    'bgp_app_peer\.py'
+    ConfigViaRestconf.Teardown_Config_Via_Restconf
+    RequestsLibrary.Delete_All_Sessions
+    SSHLibrary.Close_All_Connections
+
+Open_BGP_Peer_Console
+    [Documentation]    Create a session for BGP peer.
+    SSHLibrary.Open_Connection    ${TOOLS_SYSTEM_IP}    alias=bgp_peer_console
+    Utils.Flexible_Mininet_Login
+
+Open_BGP_Aplicationp_Peer_Console
+    [Documentation]    Create a session for BGP peer.
+    SSHLibrary.Open_Connection    ${TOOLS_SYSTEM_IP}    alias=bgp_app_peer_console
+    Utils.Flexible_Mininet_Login
+
+Start_Console_Tool
+    [Arguments]    ${command}    ${tool_opt}
+    [Documentation]    Start the tool ${command} ${tool_opt}
+    BuiltIn.Log    ${command}
+    ${output}=    SSHLibrary.Write    ${command} ${tool_opt}
+    BuiltIn.Log    ${output}
+
+Wait_Until_Console_Tool_Finish
+    [Arguments]    ${timeout}
+    [Documentation]    Wait ${timeout} for the tool exit.
+    BuiltIn.Wait Until Keyword Succeeds    ${timeout}    1s    SSHLibrary.Read Until Prompt
+
+Stop_Console_Tool
+    [Documentation]    Stop the tool if still running.
+    Utils.Write_Bare_Ctrl_C
+    ${output}=    SSHLibrary.Read    delay=1s
+    BuiltIn.Log    ${output}
+
+Read_And_Fail_If_Prompt_Is_Seen
+    [Documentation]    Try to read SSH to see prompt, but expect to see no prompt within SSHLibrary's timeout.
+    ${passed}=    BuiltIn.Run_Keyword_And_Return_Status    BuiltIn.Run_Keyword_And_Expect_Error    No match found for '${TOOLS_SYSTEM_PROMPT}' in *.    Read_Text_Before_Prompt
+    BuiltIn.Return_From_Keyword_If    ${passed}
+    BGPSpeaker.Dump_BGP_Speaker_Logs
+    Builtin.Fail    The prompt was seen but it was not expected yet
+
+Read_Text_Before_Prompt
+    [Documentation]    Log text gathered by SSHLibrary.Read_Until_Prompt.
+    ...    This needs to be a separate keyword just because how Read_And_Fail_If_Prompt_Is_Seen is implemented.
+    ${text}=    SSHLibrary.Read_Until_Prompt
+    BuiltIn.Log    ${text}
+
+Store_File_To_Workspace
+    [Arguments]    ${source_file_name}    ${target_file_name}
+    [Documentation]    Store the ${source_file_name} to the workspace as ${target_file_name}.
+    ${output_log}=    SSHLibrary.Execute_Command    cat ${source_file_name}
+    BuiltIn.Log    ${output_log}
+    Create File    ${target_file_name}    ${output_log}
+
+Check_File_For_Word_Count
+    [Arguments]    ${file_name}    ${word}    ${expected_count}
+    [Documentation]    Count ${word} in ${file_name}. Expect ${expected_count} occurence(s)
+    ${output_log}=    SSHLibrary.Execute_Command    grep -o '${word}' ${file_name} | wc -l
+    BuiltIn.Log    ${output_log}
+    BuiltIn.Should_Be_Equal_As_Strings    ${output_log}    ${expected_count}
index 6de34c9c51922528cdcbfee80e2d080728a217cf..2a7fbcca41b9b8e34aeec04b68330b75200a02b2 100644 (file)
@@ -8,5 +8,6 @@
 integration/test/csit/suites/netconf/ready/netconfready.robot
 integration/test/csit/suites/bgpcep/bgpingest/singlepeer_prefixcount.robot
 integration/test/csit/suites/bgpcep/bgpingest/singlepeer_changecount.robot
+integration/test/csit/suites/bgpcep/bgpingest/bgp_app_peer_prefixcount.robot
 integration/test/csit/suites/bgpcep/bgpingest/manypeers_prefixcount.robot
 integration/test/csit/suites/bgpcep/bgpingest/manypeers_changecount.robot
index aa292a94d17aa63621d87c74a3e56e9499c80813..2af032abd6715150a8ecb49da6ac9d7f3778f475 100755 (executable)
@@ -37,8 +37,9 @@ def _build_url(odl_ip, port, uri):
     return url
 
 
-def _build_data(xml_template, prefix_base, prefix_len, count, element="ipv4-routes"):
-    """Generate list of routes based on xml templates.
+def _stream_data(xml_template, prefix_base, prefix_len, count, element="ipv4-routes"):
+    """Stream list of routes based on xml template. Memory non-consumable
+    data generation (on the fly).
 
     Args:
         :xml_template: xml template for routes
@@ -52,41 +53,47 @@ def _build_data(xml_template, prefix_base, prefix_len, count, element="ipv4-rout
         :element: element to be returned
 
     Returns:
-        :returns xml_data: requested element as xml data
+        :yield xml_data: requested data by elements as xml data
     """
     global total_build_data_time_counter
-    build_data_timestamp = time.time()
 
     routes = md.parse(xml_template)
 
     routes_node = routes.getElementsByTagName("ipv4-routes")[0]
     route_node = routes.getElementsByTagName("ipv4-route")[0]
+    routes_node.removeChild(route_node)
+    route_prefix = route_node.getElementsByTagName("prefix")[0]
+    prefix_gap = 2 ** (32 - prefix_len)
+    prefix_index_list = range(count)
     if element == routes_node.tagName:
-        routes_node.removeChild(route_node)
-        if count:
-            prefix_gap = 2 ** (32 - prefix_len)
-
-        for prefix_index in range(count):
-            new_route_node = route_node.cloneNode(True)
-            new_route_prefix = new_route_node.getElementsByTagName("prefix")[0]
-
-            prefix = prefix_base + prefix_index * prefix_gap
-            new_route_prefix.childNodes[0].nodeValue = str(prefix) + "/" + str(prefix_len)
-
-            routes_node.appendChild(new_route_node)
-
-        xml_data = routes_node.toxml()
+        lines = routes_node.toxml().splitlines()
+        xml_head = lines[0] + "\n"
+        xml_tail = "\n".join(lines[1:])
     elif element == route_node.tagName:
-        route_node.setAttribute("xmlns", routes_node.namespaceURI)
-        route_prefix = route_node.getElementsByTagName("prefix")[0]
-        route_prefix.childNodes[0].nodeValue = str(prefix_base) + "/" + str(prefix_len)
-        xml_data = route_node.toxml()
+        xml_head = ""
+        xml_tail = ""
+        route_node.setAttribute("xmlns", route_node.namespaceURI)
     else:
-        xml_data = ""
-    routes.unlink()
-    logger.debug("xml data generated:\n%s", xml_data)
-    total_build_data_time_counter += time.time() - build_data_timestamp
-    return xml_data
+        prefix_index_list = range(0)
+
+    for prefix_index in prefix_index_list:
+        build_data_timestamp = time.time()
+        prefix = prefix_base + prefix_index * prefix_gap
+        prefix_str = str(prefix) + "/" + str(prefix_len)
+        route_prefix.childNodes[0].nodeValue = prefix_str
+        xml_data = route_node.toxml()
+        if prefix_index == 0:
+            xml_data = xml_head + xml_data
+        if prefix_index == len(prefix_index_list) - 1:
+            xml_data = xml_data + xml_tail
+        chunk = prefix_index + 1
+        if not (chunk % 1000):
+            logger.info("... streaming chunk %s (prefix: %s)", chunk, prefix_str)
+        else:
+            logger.debug("...streaming chunk %s (prefix: %s)", chunk, prefix_str)
+        logger.debug("xml data\n%s", xml_data)
+        total_build_data_time_counter += time.time() - build_data_timestamp
+        yield xml_data
 
 
 def send_request(operation, odl_ip, port, uri, auth, xml_data=None, expect_status_code=200):
@@ -125,20 +132,14 @@ def send_request(operation, odl_ip, port, uri, auth, xml_data=None, expect_statu
     except requests.exceptions.Timeout:
         logger.error("No response from %s", odl_ip)
     else:
-        logger.debug("%s %s", rsp.request, rsp.request.url)
-        logger.debug("Request headers: %s:", rsp.request.headers)
-        logger.debug("Request body: %s", rsp.request.body)
-        logger.debug("Response: %s", rsp.text)
         if rsp.status_code == expect_status_code:
             logger.debug("%s %s", rsp.request, rsp.request.url)
             logger.debug("Request headers: %s:", rsp.request.headers)
-            logger.debug("Request body: %s", rsp.request.body)
             logger.debug("Response: %s", rsp.text)
             logger.debug("%s %s", rsp, rsp.reason)
         else:
             logger.error("%s %s", rsp.request, rsp.request.url)
             logger.error("Request headers: %s:", rsp.request.headers)
-            logger.error("Request body: %s", rsp.request.body)
             logger.error("Response: %s", rsp.text)
             logger.error("%s %s", rsp, rsp.reason)
         return rsp
@@ -214,8 +215,8 @@ def post_prefixes(odl_ip, port, uri, auth, prefix_base=None, prefix_len=None,
     """
     logger.info("Post %s prefix(es) in a single request (starting from %s/%s) into %s:%s/restconf/%s",
                 count, prefix_base, prefix_len, odl_ip, port, uri)
-    xml_data = _build_data(xml_template, prefix_base, prefix_len, count)
-    send_request("POST", odl_ip, port, uri, auth, xml_data=xml_data, expect_status_code=204)
+    xml_stream = _stream_data(xml_template, prefix_base, prefix_len, count)
+    send_request("POST", odl_ip, port, uri, auth, xml_data=xml_stream, expect_status_code=204)
 
 
 def put_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count,
@@ -245,8 +246,8 @@ def put_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count,
     uri_add_prefix = uri + _uri_suffix_ipv4_routes
     logger.info("Put %s prefix(es) in a single request (starting from %s/%s) into %s:%s/restconf/%s",
                 count, prefix_base, prefix_len, odl_ip, port, uri_add_prefix)
-    xml_data = _build_data(xml_template, prefix_base, prefix_len, count)
-    send_request("PUT", odl_ip, port, uri_add_prefix, auth, xml_data=xml_data)
+    xml_stream = _stream_data(xml_template, prefix_base, prefix_len, count)
+    send_request("PUT", odl_ip, port, uri_add_prefix, auth, xml_data=xml_stream)
 
 
 def add_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count,
@@ -281,9 +282,9 @@ def add_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count,
         prefix = prefix_base + prefix_index * prefix_gap
         logger.info("Adding prefix %s/%s to %s:%s/restconf/%s",
                     prefix, prefix_len, odl_ip, port, uri)
-        xml_data = _build_data(xml_template, prefix, prefix_len, 1, "ipv4-route")
+        xml_stream = _stream_data(xml_template, prefix, prefix_len, 1, "ipv4-route")
         send_request("POST", odl_ip, port, uri_add_prefix, auth,
-                     xml_data=xml_data, expect_status_code=204)
+                     xml_data=xml_stream, expect_status_code=204)
 
 
 def delete_prefixes(odl_ip, port, uri, auth, prefix_base, prefix_len, count,
index 304bb8f1f52add7560c43893d58132ee775b2411..dcec923def4b9574f210fa7f7c2843a4b0e1a980 100755 (executable)
@@ -1086,6 +1086,7 @@ class ReadTracker(object):
         self.prefixes_introduced = 0
         self.prefixes_withdrawn = 0
         self.rx_idle_time = 0
+        self.rx_activity_detected = True
 
     def read_message_chunk(self):
         """Read up to one message
@@ -1341,20 +1342,27 @@ class ReadTracker(object):
             wait_timedelta = 0
         # And wait for event or something to read.
 
-        logger.info("total_received_update_message_counter: %s",
-                    self.updates_received)
-        logger.info("total_received_nlri_prefix_counter: %s",
-                    self.prefixes_introduced)
-        logger.info("total_received_withdrawn_prefix_counter: %s",
-                    self.prefixes_withdrawn)
+        if not self.rx_activity_detected or not (self.updates_received % 100):
+            # right time to write statistics to the log (not for every update and
+            # not too frequently to avoid having large log files)
+            logger.info("total_received_update_message_counter: %s",
+                        self.updates_received)
+            logger.info("total_received_nlri_prefix_counter: %s",
+                        self.prefixes_introduced)
+            logger.info("total_received_withdrawn_prefix_counter: %s",
+                        self.prefixes_withdrawn)
 
         start_time = time.time()
         select.select([self.socket], [], [self.socket], wait_timedelta)
         timedelta = time.time() - start_time
         self.rx_idle_time += timedelta
+        self.rx_activity_detected = timedelta < 1
 
-        logger.info("... idle for %.3fs", timedelta)
-        logger.info("total_rx_idle_time_counter: %.3fs", self.rx_idle_time)
+        if not self.rx_activity_detected or not (self.updates_received % 100):
+            # right time to write statistics to the log (not for every update and
+            # not too frequently to avoid having large log files)
+            logger.info("... idle for %.3fs", timedelta)
+            logger.info("total_rx_idle_time_counter: %.3fs", self.rx_idle_time)
         return