Jan Medved has created several benchmarking utilities for Coretutorials.
But Coretutorial project is not part of Beryllium.
The natural place for the python scripts is in tools,
as they are standalone, do not depend on anything,
and can be started from a machine remote to ODL.
Java part of benchmarking applications will be hosted elsewhere.
Change-Id: Ib872765c0608bba2296f27c4515417d8f01492be
Signed-off-by: Vratko Polak <vrpolak@cisco.com>
--- /dev/null
+#!/usr/bin/python
+__author__ = "Jan Medved"
+__copyright__ = "Copyright(c) 2015, Cisco Systems, Inc."
+__license__ = "New-style BSD"
+__email__ = "jmedved@cisco.com"
+
+import argparse
+import requests
+import json
+import csv
+import time
+
+parser = argparse.ArgumentParser(description='Datastore Benchmarking'
+ ''
+ 'See documentation @:'
+ 'https://wiki.opendaylight.org/view/Controller_Core_Functionality_Tutorials:Tutorials:Data_Store_Benchmarking_and_Data_Access_Patterns' # noqa
+ '')
+
+# Host Config
+parser.add_argument("--host", default="localhost", help="the IP of the target host to initiate benchmark testing on.")
+parser.add_argument("--port", type=int, default=8181, help="the port number of target host.")
+
+# Test Parameters
+parser.add_argument("--txtype", choices=["TX-CHAINING", "SIMPLE-TX"], nargs='+', default=["TX-CHAINING", "SIMPLE-TX"],
+ help="list of the transaction types to execute.")
+parser.add_argument("--total", type=int, default=100000, help="total number of elements to process.")
+parser.add_argument("--inner", type=int, default=[1, 10, 100, 1000, 10000, 100000],
+ help="number of inner elements to process.")
+parser.add_argument("--ops", type=int, default=[1, 10, 100, 1000, 10000, 100000],
+ help="number of operations per transaction.")
+parser.add_argument("--optype", choices=["PUT", "MERGE", "DELETE", "READ"], nargs='+',
+ default=["PUT", "MERGE", "DELETE", "READ"], help="list of the types operations to execute.")
+parser.add_argument("--format", choices=["BINDING-AWARE", "BINDING-INDEPENDENT"], nargs='+',
+ default=["BINDING-AWARE", "BINDING-INDEPENDENT"], help="list of data formats to execute.")
+parser.add_argument("--warmup", type=int, default=10, help="number of warmup runs before official test runs")
+parser.add_argument("--runs", type=int, default=10,
+ help="number of official test runs. Note: Reported results are based on these runs.")
+args = parser.parse_args()
+
+
+BASE_URL = "http://%s:%d/restconf/" % (args.host, args.port)
+
+
+def send_clear_request():
+ """
+ Sends a clear request to the dsbenchmark app. A clear will clear the test-exec data store
+ and clear the 'test-executing' flag.
+ :return: None
+ """
+ url = BASE_URL + "operations/dsbenchmark:cleanup-store"
+
+ r = requests.post(url, stream=False, auth=('admin', 'admin'))
+ print r.status_code
+
+
+def send_test_request(tx_type, operation, data_fmt, outer_elem, inner_elem, ops_per_tx):
+ """
+ Sends a request to the dsbenchmark app to start a data store benchmark test run.
+ The dsbenchmark app will perform the requested benchmark test and return measured
+ transaction times
+ :param operation: PUT, MERGE, DELETE or READ
+ :param data_fmt: BINDING-AWARE or BINDING-INDEPENDENT
+ :param outer_elem: Number of elements in the outer list
+ :param inner_elem: Number of elements in the inner list
+ :param ops_per_tx: Number of operations (PUTs, MERGEs or DELETEs) on each transaction
+ :return:
+ """
+ url = BASE_URL + "operations/dsbenchmark:start-test"
+ postheaders = {'content-type': 'application/json', 'Accept': 'application/json'}
+
+ test_request_template = '''{
+ "input": {
+ "transaction-type": "%s",
+ "operation": "%s",
+ "data-format": "%s",
+ "outerElements": %d,
+ "innerElements": %d,
+ "putsPerTx": %d
+ }
+ }'''
+ data = test_request_template % (tx_type, operation, data_fmt, outer_elem, inner_elem, ops_per_tx)
+ r = requests.post(url, data, headers=postheaders, stream=False, auth=('admin', 'admin'))
+ result = {u'http-status': r.status_code}
+ if r.status_code == 200:
+ result = dict(result.items() + json.loads(r.content)['output'].items())
+ else:
+ print 'Error %s, %s' % (r.status_code, r.content)
+ return result
+
+
+def print_results(run_type, idx, res):
+ """
+ Prints results from a dsbenchmakr test run to console
+ :param run_type: String parameter that can be used to identify the type of the
+ test run (e.g. WARMUP or TEST)
+ :param idx: Index of the test run
+ :param res: Parsed json (disctionary) that was returned from a dsbenchmark
+ test run
+ :return: None
+ """
+ print '%s #%d: status: %s, listBuildTime %d, testExecTime %d, txOk %d, txError %d' % \
+ (run_type, idx, res[u'status'], res[u'listBuildTime'], res[u'execTime'], res[u'txOk'], res[u'txError'])
+
+
+def run_test(warmup_runs, test_runs, tx_type, operation, data_fmt, outer_elem, inner_elem, ops_per_tx):
+ """
+ Execute a benchmark test. Performs the JVM 'wamrup' before the test, runs
+ the specified number of dsbenchmark test runs and computes the average time
+ for building the test data (a list of lists) and the average time for the
+ execution of the test.
+ :param warmup_runs: # of warmup runs
+ :param test_runs: # of test runs
+ :param operation: PUT, MERGE or DELETE
+ :param data_fmt: BINDING-AWARE or BINDING-INDEPENDENT
+ :param outer_elem: Number of elements in the outer list
+ :param inner_elem: Number of elements in the inner list
+ :param ops_per_tx: Number of operations (PUTs, MERGEs or DELETEs) on each transaction
+ :return: average build time AND average test execution time
+ """
+ total_build_time = 0.0
+ total_exec_time = 0.0
+
+ print 'Tx Type: {0:s}, Operation: {1:s}, Data Format: {2:s}, Outer/Inner Elements: {3:d}/{4:d}, PutsPerTx {5:d}' \
+ .format(tx_type, operation, data_fmt, outer_elem, inner_elem, ops_per_tx)
+ for idx in range(warmup_runs):
+ res = send_test_request(tx_type, operation, data_fmt, outer_elem, inner_elem, ops_per_tx)
+ print_results('WARMUP', idx, res)
+
+ for idx in range(test_runs):
+ res = send_test_request(tx_type, operation, data_fmt, outer_elem, inner_elem, ops_per_tx)
+ print_results('TEST', idx, res)
+ total_build_time += res['listBuildTime']
+ total_exec_time += res['execTime']
+
+ return total_build_time / test_runs, total_exec_time / test_runs
+
+
+if __name__ == "__main__":
+ # Test Parameters
+ TX_TYPES = args.txtype
+ TOTAL_ELEMENTS = args.total
+ INNER_ELEMENTS = args.inner
+ OPS_PER_TX = args.ops
+ OPERATIONS = args.optype
+ DATA_FORMATS = args.format
+
+ # Iterations
+ WARMUP_RUNS = args.warmup
+ TEST_RUNS = args.runs
+
+ # Clean up any data that may be present in the data store
+ send_clear_request()
+
+ # Run the benchmark tests and collect data in a csv file for import into a graphing software
+ f = open('test.csv', 'wt')
+ try:
+ start_time = time.time()
+ print "Start time: %f " % start_time
+
+ writer = csv.writer(f)
+
+ # Determine the impact of transaction type, data format and data structure on performance.
+ # Iterate over all transaction types, data formats, operation types, and different
+ # list-of-lists layouts; always use a single operation in each transaction
+ print '\n#######################################'
+ print 'Tx type, data format & data structure'
+ print '#######################################'
+ for tx_type in TX_TYPES:
+ print '***************************************'
+ print 'Transaction Type: %s' % tx_type
+ print '***************************************'
+ writer.writerow((('%s:' % tx_type), '', ''))
+
+ for fmt in DATA_FORMATS:
+ print '---------------------------------------'
+ print 'Data format: %s' % fmt
+ print '---------------------------------------'
+ writer.writerow(('', ('%s:' % fmt), ''))
+
+ for oper in OPERATIONS:
+ print 'Operation: %s' % oper
+ writer.writerow(('', '', '%s:' % oper))
+
+ for elem in INNER_ELEMENTS:
+ avg_build_time, avg_exec_time = \
+ run_test(WARMUP_RUNS, TEST_RUNS, tx_type, oper, fmt, TOTAL_ELEMENTS / elem, elem, 1)
+ e_label = '%d/%d' % (TOTAL_ELEMENTS / elem, elem)
+ writer.writerow(('', '', '', e_label, avg_build_time, avg_exec_time,
+ (avg_build_time + avg_exec_time)))
+
+ # Determine the impact of number of writes per transaction on performance.
+ # Iterate over all transaction types, data formats, operation types, and
+ # operations-per-transaction; always use a list of lists where the inner list has one parameter
+ print '\n#######################################'
+ print 'Puts per tx'
+ print '#######################################'
+ for tx_type in TX_TYPES:
+ print '***************************************'
+ print 'Transaction Type: %s' % tx_type
+ print '***************************************'
+ writer.writerow((('%s:' % tx_type), '', ''))
+
+ for fmt in DATA_FORMATS:
+ print '---------------------------------------'
+ print 'Data format: %s' % fmt
+ print '---------------------------------------'
+ writer.writerow(('', ('%s:' % fmt), ''))
+
+ for oper in OPERATIONS:
+ print 'Operation: %s' % oper
+ writer.writerow(('', '', '%s:' % oper))
+
+ for wtx in OPS_PER_TX:
+ avg_build_time, avg_exec_time = \
+ run_test(WARMUP_RUNS, TEST_RUNS, tx_type, oper, fmt, TOTAL_ELEMENTS, 1, wtx)
+ writer.writerow(('', '', '', wtx, avg_build_time, avg_exec_time,
+ (avg_build_time + avg_exec_time)))
+
+ end_time = time.time()
+ print "End time: %f " % end_time
+ print "Total execution time: %f" % (end_time - start_time)
+
+ finally:
+ f.close()
--- /dev/null
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Cisco Systems All rights reserved.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License v1.0 which accompanies this distribution,
+# and is available at http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+__author__ = "Jan Medved"
+__copyright__ = "Copyright(c) 2015, Cisco Systems, Inc."
+__license__ = "Eclipse Public License v1.0"
+__email__ = "jmedved@cisco.com"
+
+import argparse
+import requests
+import json
+import csv
+
+global BASE_URL
+
+
+def send_test_request(producer_type, producers, listeners, payload_size, iterations):
+ """
+ Sends a request to the rpcbenchmark app to start a data store benchmark test run.
+ The rpcbenchmark app will perform the requested benchmark test and return measured
+ test execution time and RPC throughput
+
+ :param operation: operation type
+ :param clients: number of simulated RPC clients
+ :param servers: Number of simulated RPC servers if operation type is ROUTED-***
+ :param payload_size: Payload size for the test RPCs
+ :param iterations: Number of iterations to run
+ :return: Result from the test request REST call (json)
+ """
+ url = BASE_URL + "operations/ntfbenchmark:start-test"
+ postheaders = {'content-type': 'application/json', 'Accept': 'application/json'}
+
+ test_request_template = '''{
+ "input": {
+ "producer-type": "%s",
+ "producers": "%s",
+ "listeners": "%s",
+ "payload-size": "%s",
+ "iterations": "%s"
+ }
+ }'''
+ data = test_request_template % (producer_type, producers, listeners, payload_size, iterations)
+ r = requests.post(url, data, headers=postheaders, stream=False, auth=('admin', 'admin'))
+ result = {u'http-status': r.status_code}
+ if r.status_code == 200:
+ result = dict(result.items() + json.loads(r.content)['output'].items())
+ else:
+ print 'Error %s, %s' % (r.status_code, r.content)
+ return result
+
+
+def print_results(run_type, idx, res):
+ """
+ Prints results from a dsbenchmakr test run to console
+ :param run_type: String parameter that can be used to identify the type of the
+ test run (e.g. WARMUP or TEST)
+ :param idx: Index of the test run
+ :param res: Parsed json (disctionary) that was returned from a dsbenchmark
+ test run
+ :return: None
+ """
+ print '%s #%d: ProdOk: %d, ProdError: %d, LisOk: %d, ProdRate: %d, LisRate %d, ProdTime: %d, ListTime %d' % \
+ (run_type, idx,
+ res[u'producer-ok'], res[u'producer-error'], res[u'listener-ok'], res[u'producer-rate'],
+ res[u'listener-rate'], res[u'producer-elapsed-time'], res[u'listener-elapsed-time'])
+
+
+def run_test(warmup_runs, test_runs, producer_type, producers, listeners, payload_size, iterations):
+ """
+ Execute a benchmark test. Performs the JVM 'wamrup' before the test, runs
+ the specified number of dsbenchmark test runs and computes the average time
+ for building the test data (a list of lists) and the average time for the
+ execution of the test.
+ :param warmup_runs: # of warmup runs
+ :param test_runs: # of test runs
+ :param operation: PUT, MERGE or DELETE
+ :param data_fmt: BINDING-AWARE or BINDING-INDEPENDENT
+ :param outer_elem: Number of elements in the outer list
+ :param inner_elem: Number of elements in the inner list
+ :param ops_per_tx: Number of operations (PUTs, MERGEs or DELETEs) on each transaction
+ :return: average build time AND average test execution time
+ """
+ total_exec_time = 0.0
+ total_prate = 0.0
+ total_lrate = 0.0
+
+ for idx in range(warmup_runs):
+ res = send_test_request(producer_type, producers, listeners, payload_size, iterations)
+ print_results('WARM-UP', idx, res)
+
+ for idx in range(test_runs):
+ res = send_test_request(producer_type, producers, listeners, payload_size, iterations)
+ print_results('TEST', idx, res)
+ total_exec_time += res['listener-elapsed-time']
+ total_prate += res['producer-rate']
+ total_lrate += res['listener-rate']
+
+ return total_exec_time / test_runs, total_prate / test_runs, total_lrate / test_runs
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description='RPC Benchmarking')
+
+ # Host Config
+ parser.add_argument("--host", default="localhost", help="IP of the target host where benchmarks will be run.")
+ parser.add_argument("--port", type=int, default=8181, help="The port number of target host.")
+
+ # Test Parameters
+ parser.add_argument("--ptype", choices=["DROPPING", "BLOCKING"], nargs='+', default='BLOCKING',
+ help='Producer type. (default: BLOCKING)')
+ parser.add_argument("--warm", type=int, default=10, help='The number of warm-up runs before the measured test runs'
+ '(Default 10)')
+ parser.add_argument("--run", type=int, default=10,
+ help='The number of measured test runs. Reported results are based on these average of all'
+ " measured runs. (Default 10)")
+ parser.add_argument("--producers", type=int, nargs='+', default=[1, 2, 4, 8, 16, 32],
+ help='The number of test producers to start. (Default 10)')
+ parser.add_argument("--listeners", type=int, nargs='+', default=[1, 2, 4, 8, 16, 32],
+ help='The number of test listeners to start. (Default 10)')
+ parser.add_argument("--iterations", type=int, default=100, help='The number requests that each producer issues '
+ 'during the test run. (Default 10)')
+ parser.add_argument("--payload", type=int, default=10, help='Payload size for the RPC - number of elements in a '
+ 'simple integer list. (Default 10)')
+
+ args = parser.parse_args()
+ BASE_URL = "http://%s:%d/restconf/" % (args.host, args.port)
+
+ # Run the benchmark tests and collect data in a csv file for import into a graphing software
+ f = open('test.csv', 'wt')
+ try:
+ writer = csv.writer(f)
+ lrate_matrix = []
+ prate_matrix = []
+ for prod in args.producers:
+ lrate_row = ['']
+ prate_row = ['']
+ for lis in args.listeners:
+ exec_time, prate, lrate = run_test(args.warm, args.run, args.ptype, prod, lis,
+ args.payload, args.iterations)
+ print 'Producers: %d, Listeners: %d, prate: %d, lrate: %d' % (prod, lis, prate, lrate)
+ lrate_row.append(lrate)
+ prate_row.append(prate)
+
+ lrate_matrix.append(lrate_row)
+ prate_matrix.append(prate_row)
+
+ print lrate_matrix
+ print prate_matrix
+
+ # writer.writerow((('%s:' % args.ptype), '', '', ''))
+ # writer.writerow(('', exec_time, prate, lrate))
+
+ writer.writerow(('Listener Rates:', ''))
+ writer.writerows(lrate_matrix)
+ writer.writerow(('Producer Rates:', ''))
+ writer.writerows(prate_matrix)
+
+ finally:
+ f.close()
--- /dev/null
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Cisco Systems All rights reserved.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License v1.0 which accompanies this distribution,
+# and is available at http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+__author__ = "Jan Medved"
+__copyright__ = "Copyright(c) 2015, Cisco Systems, Inc."
+__license__ = "Eclipse Public License v1.0"
+__email__ = "jmedved@cisco.com"
+
+import argparse
+import requests
+import json
+import csv
+
+global BASE_URL
+
+
+def send_test_request(operation, clients, servers, payload_size, iterations):
+ """
+ Sends a request to the rpcbenchmark app to start a data store benchmark test run.
+ The rpcbenchmark app will perform the requested benchmark test and return measured
+ test execution time and RPC throughput
+
+ :param operation: operation type
+ :param clients: number of simulated RPC clients
+ :param servers: Number of simulated RPC servers if operation type is ROUTED-***
+ :param payload_size: Payload size for the test RPCs
+ :param iterations: Number of iterations to run
+ :return: Result from the test request REST call (json)
+ """
+ url = BASE_URL + "operations/rpcbenchmark:start-test"
+ postheaders = {'content-type': 'application/json', 'Accept': 'application/json'}
+
+ test_request_template = '''{
+ "input": {
+ "operation": "%s",
+ "num-clients": "%s",
+ "num-servers": "%s",
+ "payload-size": "%s",
+ "iterations": "%s"
+ }
+ }'''
+ data = test_request_template % (operation, clients, servers, payload_size, iterations)
+ r = requests.post(url, data, headers=postheaders, stream=False, auth=('admin', 'admin'))
+ result = {u'http-status': r.status_code}
+ if r.status_code == 200:
+ result = dict(result.items() + json.loads(r.content)['output'].items())
+ else:
+ print 'Error %s, %s' % (r.status_code, r.content)
+ return result
+
+
+def print_results(run_type, idx, res):
+ """
+ Prints results from a dsbenchmakr test run to console
+ :param run_type: String parameter that can be used to identify the type of the
+ test run (e.g. WARMUP or TEST)
+ :param idx: Index of the test run
+ :param res: Parsed json (disctionary) that was returned from a dsbenchmark
+ test run
+ :return: None
+ """
+ print '%s #%d: Ok: %d, Error: %d, Rate: %d, Exec time: %d' % \
+ (run_type, idx,
+ res[u'global-rtc-client-ok'], res[u'global-rtc-client-error'], res[u'rate'], res[u'exec-time'])
+
+
+def run_test(warmup_runs, test_runs, operation, clients, servers, payload_size, iterations):
+ """
+ Execute a benchmark test. Performs the JVM 'wamrup' before the test, runs
+ the specified number of dsbenchmark test runs and computes the average time
+ for building the test data (a list of lists) and the average time for the
+ execution of the test.
+ :param warmup_runs: # of warmup runs
+ :param test_runs: # of test runs
+ :param operation: PUT, MERGE or DELETE
+ :param data_fmt: BINDING-AWARE or BINDING-INDEPENDENT
+ :param outer_elem: Number of elements in the outer list
+ :param inner_elem: Number of elements in the inner list
+ :param ops_per_tx: Number of operations (PUTs, MERGEs or DELETEs) on each transaction
+ :return: average build time AND average test execution time
+ """
+ total_exec_time = 0.0
+ total_rate = 0.0
+
+ for idx in range(warmup_runs):
+ res = send_test_request(operation, clients, servers, payload_size, iterations)
+ print_results('WARM-UP', idx, res)
+
+ for idx in range(test_runs):
+ res = send_test_request(operation, clients, servers, payload_size, iterations)
+ print_results('TEST', idx, res)
+ total_exec_time += res['exec-time']
+ total_rate += res['rate']
+
+ return total_exec_time / test_runs, total_rate / test_runs
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description='RPC Benchmarking')
+
+ # Host Config
+ parser.add_argument("--host", default="localhost", help="IP of the target host where benchmarks will be run.")
+ parser.add_argument("--port", type=int, default=8181, help="The port number of target host.")
+
+ # Test Parameters
+ parser.add_argument("--operation", choices=["GLOBAL-RTC", "ROUTED-RTC"], default='GLOBAL-RTC',
+ help='RPC and client type. RPC can be global or routcan be run-to-completion (RTC).'
+ '(default: GLOBAL-RTC - Global RPC, Run-to-completion client)')
+ parser.add_argument("--warm", type=int, default=10, help='The number of warm-up runs before the measured test runs'
+ '(Default 10)')
+ parser.add_argument("--run", type=int, default=10,
+ help='The number of measured test runs. Reported results are based on these average of all'
+ " measured runs. (Default 10)")
+ parser.add_argument("--clients", type=int, nargs='+', default=[1, 2, 4, 8, 16, 32, 64],
+ help='The number of test RPC clients to start. (Default 10)')
+ parser.add_argument("--servers", type=int, nargs='+', default=[1, 2, 4, 8, 16, 32, 64],
+ help='The number of routed RPC servers to start in the routed RPC test. Ignored in the global '
+ 'RPC test. (Default 10)')
+ parser.add_argument("--iterations", type=int, default=10, help='The number requests that each RPC client issues '
+ 'during the test run. (Default 10)')
+ parser.add_argument("--payload", type=int, default=10, help='Payload size for the RPC - number of elements in a '
+ 'simple integer list. (Default 10)')
+
+ args = parser.parse_args()
+ BASE_URL = "http://%s:%d/restconf/" % (args.host, args.port)
+
+ if args.operation == 'GLOBAL-RTC':
+ servers = [1]
+ else:
+ servers = args.servers
+
+ # Run the benchmark tests and collect data in a csv file for import into a graphing software
+ f = open('test.csv', 'wt')
+ try:
+ writer = csv.writer(f)
+ rate_matrix = []
+
+ for svr in servers:
+ rate_row = ['']
+ for client in args.clients:
+ exec_time, rate = \
+ run_test(args.warm, args.run, args.operation, client, svr, args.payload, args.iterations)
+ rate_row.append(rate)
+ rate_matrix.append(rate_row)
+ print rate_matrix
+
+ writer.writerow(('RPC Rates:', ''))
+ writer.writerows(rate_matrix)
+ finally:
+ f.close()
Code Review / integration / test.git / commitdiff