From eca85df6de7c5383c77b3bd55d4dcc2354878af2 Mon Sep 17 00:00:00 2001 From: "miroslav.kovac" Date: Tue, 22 Mar 2016 11:12:34 +0100 Subject: [PATCH] Add a batching system in Netconf testtool We can now set how many devices will be configured in one request. It can be done asynchronously or synchronously. It s configured using json file instead of XML. Change-Id: Ic4d666b21771e8cebdaaa1fabb246ac482bd8ef9 Signed-off-by: miroslav.kovac --- .../netconf/test/tool/Execution.java | 28 +++- .../netconf/test/tool/TesttoolParameters.java | 157 +++++++++++++----- .../src/main/resources/config-template.json | 11 ++ 3 files changed, 145 insertions(+), 51 deletions(-) create mode 100644 netconf/tools/netconf-testtool/src/main/resources/config-template.json diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java index 5fc8d9ba9f..e2799d8661 100644 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java +++ b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java @@ -64,9 +64,9 @@ public class Execution implements Callable { this.payloads = new ArrayList<>(); for (DestToPayload payload : payloads) { - AsyncHttpClient.BoundRequestBuilder requestBuilder = asyncHttpClient.preparePut(payload.getDestination()) - .addHeader("Content-Type", "application/xml") - .addHeader("Accept", "application/xml") + AsyncHttpClient.BoundRequestBuilder requestBuilder = asyncHttpClient.preparePost(payload.getDestination()) + .addHeader("Content-Type", "application/json") + .addHeader("Accept", "application/json") .setBody(payload.getPayload()) .setRequestTimeout(Integer.MAX_VALUE); @@ -75,7 +75,7 @@ public class Execution implements Callable { .setScheme(Realm.AuthScheme.BASIC) .setPrincipal(params.auth.get(0)) .setPassword(params.auth.get(1)) - .setMethodName("PUT") + .setMethodName("POST") .setUsePreemptiveAuth(true) .build()); } @@ -89,9 +89,14 @@ public class Execution implements Callable { try { Response response = asyncHttpClient.executeRequest(request).get(); if (response.getStatusCode() != 200 && response.getStatusCode() != 204) { - LOG.warn("Status code: {}", response.getStatusCode()); - LOG.warn("url: {}", request.getUrl()); - LOG.warn(response.getResponseBody()); + if (response.getStatusCode() == 409) { + LOG.warn("Request failed, status code: {} - one or more of the devices" + + " is already configured, skipping the whole batch", response.getStatusCode()); + } else { + LOG.warn("Status code: {}", response.getStatusCode()); + LOG.warn("url: {}", request.getUrl()); + LOG.warn(response.getResponseBody()); + } } } catch (InterruptedException | ExecutionException | IOException e) { LOG.warn(e.toString()); @@ -115,8 +120,13 @@ public class Execution implements Callable { public STATE onStatusReceived(HttpResponseStatus status) throws Exception { super.onStatusReceived(status); if (status.getStatusCode() != 200 && status.getStatusCode() != 204) { - LOG.warn("Request failed, status code: {}", status.getStatusCode() + status.getStatusText()); - LOG.warn("request: {}", request.toString()); + if (status.getStatusCode() == 409) { + LOG.warn("Request failed, status code: {} - one or more of the devices" + + " is already configured, skipping the whole batch", status.getStatusCode()); + } else { + LOG.warn("Request failed, status code: {}", status.getStatusCode() + status.getStatusText()); + LOG.warn("request: {}", request.toString()); + } } return STATE.CONTINUE; } diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java index eb6363227d..1d8eda8f78 100644 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java +++ b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved. + * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -33,73 +34,52 @@ public class TesttoolParameters { private static final String PORT_KEY = "{PORT}"; private static final String SSH = "{SSH}"; private static final String ADDRESS_PORT = "{ADDRESS:PORT}"; - private static final String dest = "http://{ADDRESS:PORT}/restconf/config/network-topology:network-topology/topology/topology-netconf/node/{PORT}-sim-device"; - - private static final String RESOURCE = "/config-template.xml"; - private InputStream stream; + private static final String dest = "http://{ADDRESS:PORT}/restconf/config/network-topology:network-topology/topology/topology-netconf/"; + private static final String RESOURCE = "/config-template.json"; @Arg(dest = "edit-content") public File editContent; - @Arg(dest = "async") public boolean async; - @Arg(dest = "thread-amount") public int threadAmount; - @Arg(dest = "throttle") public int throttle; - @Arg(dest = "auth") public ArrayList auth; - @Arg(dest = "controller-destination") public String controllerDestination; - @Arg(dest = "schemas-dir") public File schemasDir; - @Arg(dest = "devices-count") public int deviceCount; - @Arg(dest = "devices-per-port") public int devicesPerPort; - @Arg(dest = "starting-port") public int startingPort; - @Arg(dest = "generate-config-connection-timeout") public int generateConfigsTimeout; - @Arg(dest = "generate-config-address") public String generateConfigsAddress; - @Arg(dest = "distro-folder") public File distroFolder; - @Arg(dest = "generate-configs-batch-size") public int generateConfigBatchSize; - @Arg(dest = "ssh") public boolean ssh; - @Arg(dest = "exi") public boolean exi; - @Arg(dest = "debug") public boolean debug; - @Arg(dest = "notification-file") public File notificationFile; - @Arg(dest = "md-sal") public boolean mdSal; - @Arg(dest = "initial-config-xml-file") public File initialConfigXMLFile; - @Arg(dest = "time-out") public long timeOut; + private InputStream stream; static ArgumentParser getParser() { final ArgumentParser parser = ArgumentParsers.newArgumentParser("netconf testtool"); @@ -134,10 +114,10 @@ public class TesttoolParameters { parser.addArgument("--controller-destination") .type(String.class) - .help("Ip address and port of controller. Must be in following format : "+ - "if available it will be used for spawning netconf connectors via topology configuration as "+ - "a part of URI. Example (http:///restconf/config/network-topology:network-topology/topology/topology-netconf/node/)"+ - "otherwise it will just start simulated devices and skip the execution of PUT requests") + .help("Ip address and port of controller. Must be in following format : " + + "if available it will be used for spawning netconf connectors via topology configuration as " + + "a part of URI. Example (http:///restconf/config/network-topology:network-topology/topology/topology-netconf/node/)" + + "otherwise it will just start simulated devices and skip the execution of PUT requests") .dest("controller-destination"); parser.addArgument("--device-count") @@ -187,7 +167,7 @@ public class TesttoolParameters { parser.addArgument("--generate-configs-batch-size") .type(Integer.class) - .setDefault(4000) + .setDefault(1) .help("Number of connector configs per generated file") .dest("generate-configs-batch-size"); @@ -242,6 +222,25 @@ public class TesttoolParameters { return null; } + private static String modifyMessage(final StringBuilder payloadBuilder, final int payloadPosition, final int size) { + if (size == 1) { + return payloadBuilder.toString(); + } + + if (payloadPosition == 0) { + payloadBuilder.insert(payloadBuilder.toString().indexOf('{', 2), "["); + payloadBuilder.replace(payloadBuilder.length() - 1, payloadBuilder.length(), ","); + } else if (payloadPosition + 1 == size) { + payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1); + payloadBuilder.insert(payloadBuilder.toString().indexOf('}', 2) + 1, "]"); + } else { + payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1); + payloadBuilder.replace(payloadBuilder.length() - 2, payloadBuilder.length() - 1, ","); + payloadBuilder.deleteCharAt(payloadBuilder.toString().lastIndexOf('}')); + } + return payloadBuilder.toString(); + } + void validate() { if (editContent == null) { stream = TesttoolParameters.class.getResourceAsStream(RESOURCE); @@ -267,11 +266,10 @@ public class TesttoolParameters { } } - public ArrayList> getThreadsPayloads(List openDevices) { + public ArrayList> getThreadsPayloads(final List openDevices) { final String editContentString; try { - if(stream == null) - { + if (stream == null) { editContentString = Files.toString(editContent, Charsets.UTF_8); } else { editContentString = CharStreams.toString(new InputStreamReader(stream, Charsets.UTF_8)); @@ -280,18 +278,67 @@ public class TesttoolParameters { throw new IllegalArgumentException("Cannot read content of " + editContent); } + int from, to; + Iterator iterator; + final ArrayList> allThreadsPayloads = new ArrayList<>(); - for (int i = 0; i < threadAmount; i++) { - final ArrayList payloads = new ArrayList<>(); - for (int j = 0; j < openDevices.size(); j++) { - final StringBuilder destBuilder = new StringBuilder(dest); - destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT), destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(), controllerDestination) - .replace(destBuilder.indexOf(PORT_KEY), destBuilder.indexOf(PORT_KEY) + PORT_KEY.length(), Integer.toString(openDevices.get(j))); - payloads.add(new Execution.DestToPayload(destBuilder.toString(), prepareMessage(openDevices.get(j), editContentString))); + if (generateConfigBatchSize > 1) { + + final int batchedRequests = openDevices.size() / generateConfigBatchSize; + final int batchedRequestsPerThread = batchedRequests / threadAmount; + final int leftoverBatchedRequests = (batchedRequests) % threadAmount; + final int leftoverRequests = openDevices.size() - (batchedRequests * generateConfigBatchSize); + + final StringBuilder destBuilder = new StringBuilder(dest); + destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT), destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(), controllerDestination); + + for (int l = 0; l < threadAmount; l++) { + from = l * (batchedRequests * batchedRequestsPerThread); + to = from + (batchedRequests * batchedRequestsPerThread); + iterator = openDevices.subList(from, to).iterator(); + allThreadsPayloads.add(createBatchedPayloads(batchedRequestsPerThread, iterator, editContentString, destBuilder.toString())); + } + ArrayList payloads = null; + if (leftoverBatchedRequests > 0) { + from = threadAmount * (batchedRequests * batchedRequestsPerThread); + to = from + (batchedRequests * batchedRequestsPerThread); + iterator = openDevices.subList(from, to).iterator(); + payloads = createBatchedPayloads(leftoverBatchedRequests, iterator, editContentString, destBuilder.toString()); + } + String payload = ""; + + for (int j = 0; j < leftoverRequests; j++) { + from = openDevices.size() - leftoverRequests; + to = openDevices.size(); + iterator = openDevices.subList(from, to).iterator(); + final StringBuilder payloadBuilder = new StringBuilder(prepareMessage(iterator.next(), editContentString)); + payload += modifyMessage(payloadBuilder, j, leftoverRequests); + } + if (leftoverRequests > 0 || leftoverBatchedRequests > 0) { + + if (payloads != null) { + payloads.add(new Execution.DestToPayload(destBuilder.toString(), payload)); + } + allThreadsPayloads.add(payloads); + } + } else { + final int requestPerThreads = openDevices.size() / threadAmount; + final int leftoverRequests = openDevices.size() % threadAmount; + + for (int i = 0; i < threadAmount; i++) { + from = i * requestPerThreads; + to = from + requestPerThreads; + iterator = openDevices.subList(from, to).iterator(); + allThreadsPayloads.add(createPayloads(iterator, editContentString)); } - allThreadsPayloads.add(payloads); - } + if (leftoverRequests > 0) { + from = (threadAmount) * requestPerThreads; + to = from + leftoverRequests; + iterator = openDevices.subList(from, to).iterator(); + allThreadsPayloads.add(createPayloads(iterator, editContentString)); + } + } return allThreadsPayloads; } @@ -310,4 +357,30 @@ public class TesttoolParameters { } return messageBuilder.toString(); } + + private ArrayList createPayloads(final Iterator openDevices, final String editContentString) { + final ArrayList payloads = new ArrayList<>(); + + while (openDevices.hasNext()) { + final StringBuilder destBuilder = new StringBuilder(dest); + destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT), destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(), controllerDestination); + payloads.add(new Execution.DestToPayload(destBuilder.toString(), prepareMessage(openDevices.next(), editContentString))); + } + return payloads; + } + + private ArrayList createBatchedPayloads(final int batchedRequestsCount, final Iterator openDevices, final String editContentString, + final String destination) { + final ArrayList payloads = new ArrayList<>(); + + for (int i = 0; i < batchedRequestsCount; i++) { + String payload = ""; + for (int j = 0; j < generateConfigBatchSize; j++) { + final StringBuilder payloadBuilder = new StringBuilder(prepareMessage(openDevices.next(), editContentString)); + payload += modifyMessage(payloadBuilder, j, generateConfigBatchSize); + } + payloads.add(new Execution.DestToPayload(destination, payload)); + } + return payloads; + } } diff --git a/netconf/tools/netconf-testtool/src/main/resources/config-template.json b/netconf/tools/netconf-testtool/src/main/resources/config-template.json new file mode 100644 index 0000000000..479f20d678 --- /dev/null +++ b/netconf/tools/netconf-testtool/src/main/resources/config-template.json @@ -0,0 +1,11 @@ +{ + "node": { + "node-id": "{PORT}-sim-device", + "host": "{HOST}", + "port": "{PORT}", + "username": "admin", + "password": "admin", + "tcp-only": "{SSH}", + "keepalive-delay": "0" + } +} \ No newline at end of file -- 2.36.6