2 ##############################################################################
3 # Copyright (c) 2015 Cisco Systems All rights reserved.
5 # This program and the accompanying materials are made available under the
6 # terms of the Eclipse Public License v1.0 which accompanies this distribution,
7 # and is available at http://www.eclipse.org/legal/epl-v10.html
8 ##############################################################################
16 __author__ = "Jan Medved"
17 __copyright__ = "Copyright(c) 2015, Cisco Systems, Inc."
18 __license__ = "Eclipse Public License v1.0"
19 __email__ = "jmedved@cisco.com"
25 def send_test_request(producer_type, producers, listeners, payload_size, iterations):
27 Sends a request to the rpcbenchmark app to start a data store benchmark test run.
28 The rpcbenchmark app will perform the requested benchmark test and return measured
29 test execution time and RPC throughput
31 :param operation: operation type
32 :param clients: number of simulated RPC clients
33 :param servers: Number of simulated RPC servers if operation type is ROUTED-***
34 :param payload_size: Payload size for the test RPCs
35 :param iterations: Number of iterations to run
36 :return: Result from the test request REST call (json)
38 url = BASE_URL + "operations/ntfbenchmark:start-test"
39 postheaders = {'content-type': 'application/json', 'Accept': 'application/json'}
41 test_request_template = '''{
43 "producer-type": "%s",
50 data = test_request_template % (producer_type, producers, listeners, payload_size, iterations)
51 r = requests.post(url, data, headers=postheaders, stream=False, auth=('admin', 'admin'))
52 result = {u'http-status': r.status_code}
53 if r.status_code == 200:
54 result = dict(result.items() + json.loads(r.content)['output'].items())
56 print 'Error %s, %s' % (r.status_code, r.content)
60 def print_results(run_type, idx, res):
62 Prints results from a dsbenchmakr test run to console
63 :param run_type: String parameter that can be used to identify the type of the
64 test run (e.g. WARMUP or TEST)
65 :param idx: Index of the test run
66 :param res: Parsed json (disctionary) that was returned from a dsbenchmark
70 print '%s #%d: ProdOk: %d, ProdError: %d, LisOk: %d, ProdRate: %d, LisRate %d, ProdTime: %d, ListTime %d' % \
72 res[u'producer-ok'], res[u'producer-error'], res[u'listener-ok'], res[u'producer-rate'],
73 res[u'listener-rate'], res[u'producer-elapsed-time'], res[u'listener-elapsed-time'])
76 def run_test(warmup_runs, test_runs, producer_type, producers, listeners, payload_size, iterations):
78 Execute a benchmark test. Performs the JVM 'wamrup' before the test, runs
79 the specified number of dsbenchmark test runs and computes the average time
80 for building the test data (a list of lists) and the average time for the
81 execution of the test.
82 :param warmup_runs: # of warmup runs
83 :param test_runs: # of test runs
84 :param operation: PUT, MERGE or DELETE
85 :param data_fmt: BINDING-AWARE or BINDING-INDEPENDENT
86 :param outer_elem: Number of elements in the outer list
87 :param inner_elem: Number of elements in the inner list
88 :param ops_per_tx: Number of operations (PUTs, MERGEs or DELETEs) on each transaction
89 :return: average build time AND average test execution time
95 for idx in range(warmup_runs):
96 res = send_test_request(producer_type, producers, listeners, payload_size, iterations)
97 print_results('WARM-UP', idx, res)
99 for idx in range(test_runs):
100 res = send_test_request(producer_type, producers, listeners, payload_size, iterations)
101 print_results('TEST', idx, res)
102 total_exec_time += res['listener-elapsed-time']
103 total_prate += res['producer-rate']
104 total_lrate += res['listener-rate']
106 return total_exec_time / test_runs, total_prate / test_runs, total_lrate / test_runs
109 if __name__ == "__main__":
110 parser = argparse.ArgumentParser(description='RPC Benchmarking')
113 parser.add_argument("--host", default="localhost", help="IP of the target host where benchmarks will be run.")
114 parser.add_argument("--port", type=int, default=8181, help="The port number of target host.")
117 parser.add_argument("--ptype", choices=["DROPPING", "BLOCKING"], nargs='+', default='BLOCKING',
118 help='Producer type. (default: BLOCKING)')
119 parser.add_argument("--warm", type=int, default=10, help='The number of warm-up runs before the measured test runs'
121 parser.add_argument("--run", type=int, default=10,
122 help='The number of measured test runs. Reported results are based on these average of all'
123 " measured runs. (Default 10)")
124 parser.add_argument("--producers", type=int, nargs='+', default=[1, 2, 4, 8, 16, 32],
125 help='The number of test producers to start. (Default 10)')
126 parser.add_argument("--listeners", type=int, nargs='+', default=[1, 2, 4, 8, 16, 32],
127 help='The number of test listeners to start. (Default 10)')
128 parser.add_argument("--iterations", type=int, default=100, help='The number requests that each producer issues '
129 'during the test run. (Default 10)')
130 parser.add_argument("--payload", type=int, default=10, help='Payload size for the RPC - number of elements in a '
131 'simple integer list. (Default 10)')
133 args = parser.parse_args()
134 BASE_URL = "http://%s:%d/restconf/" % (args.host, args.port)
136 # Run the benchmark tests and collect data in a csv file for import into a graphing software
137 f = open('test.csv', 'wt')
139 writer = csv.writer(f)
142 for prod in args.producers:
145 for lis in args.listeners:
146 exec_time, prate, lrate = run_test(args.warm, args.run, args.ptype, prod, lis,
147 args.payload, args.iterations)
148 print 'Producers: %d, Listeners: %d, prate: %d, lrate: %d' % (prod, lis, prate, lrate)
149 lrate_row.append(lrate)
150 prate_row.append(prate)
152 lrate_matrix.append(lrate_row)
153 prate_matrix.append(prate_row)
158 # writer.writerow((('%s:' % args.ptype), '', '', ''))
159 # writer.writerow(('', exec_time, prate, lrate))
161 writer.writerow(('Listener Rates:', ''))
162 writer.writerows(lrate_matrix)
163 writer.writerow(('Producer Rates:', ''))
164 writer.writerows(prate_matrix)