From 13df03aa3e4ea0b1bb82020e219227f76bb88104 Mon Sep 17 00:00:00 2001 From: Illia Date: Mon, 22 Feb 2021 13:39:30 +0200 Subject: [PATCH] Fix testtool device registration The NETCONF testtool functionality for automatic bulk registration of devices was broken for some time. We have decided to not only fix the problem but also refactor testtool the following way: - Create request bodies using NormalizedNodes instead of using config templates. - Register devices with PATCH instead of POST to allow merging nodes into topology and prevent error for consequent batch requests. JIRA: NETCONF-796 Change-Id: Ifd7c74e9b07bb56b84e6a1ffea0bf2809b85246c Signed-off-by: Illia Signed-off-by: Ivan Hrasko --- docs/testtool.rst | 79 +++++--- .../netconf/test/tool/Execution.java | 155 +++++++-------- .../opendaylight/netconf/test/tool/Main.java | 34 ++-- .../netconf/test/tool/PayloadCreator.java | 175 +++++++++++++++++ .../netconf/test/tool/TesttoolParameters.java | 181 +----------------- .../netconf/test/tool/model/Node.java | 99 ---------- .../netconf/test/tool/model/Payload.java | 28 --- .../netconf/test/tool/model/Topology.java | 47 ----- .../src/main/resources/config-template.json | 11 -- .../src/main/resources/config-template.xml | 9 - 10 files changed, 326 insertions(+), 492 deletions(-) create mode 100644 netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/PayloadCreator.java delete mode 100644 netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Node.java delete mode 100644 netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Payload.java delete mode 100644 netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Topology.java delete mode 100644 netconf/tools/netconf-testtool/src/main/resources/config-template.json delete mode 100644 netconf/tools/netconf-testtool/src/main/resources/config-template.xml diff --git a/docs/testtool.rst b/docs/testtool.rst index 4327d978b2..af599e5dd6 100644 --- a/docs/testtool.rst +++ b/docs/testtool.rst @@ -145,14 +145,22 @@ Testtool help :: - usage: netconf testtool [-h] [--edit-content EDIT-CONTENT] [--async-requests {true,false}] [--thread-amount THREAD-AMOUNT] [--throttle THROTTLE] - [--auth AUTH AUTH] [--controller-destination CONTROLLER-DESTINATION] [--device-count DEVICES-COUNT] - [--devices-per-port DEVICES-PER-PORT] [--schemas-dir SCHEMAS-DIR] [--notification-file NOTIFICATION-FILE] - [--initial-config-xml-file INITIAL-CONFIG-XML-FILE] [--starting-port STARTING-PORT] + usage: netconf testtool [-h] [--edit-content EDIT-CONTENT] [--async-requests {true,false}] + [--thread-amount THREAD-AMOUNT] [--throttle THROTTLE] + [--controller-auth-username CONTROLLER-AUTH-USERNAME] + [--controller-auth-password CONTROLLER-AUTH-PASSWORD] + [--controller-ip CONTROLLER-IP] [--controller-port CONTROLLER-PORT] + [--device-count DEVICES-COUNT] [--devices-per-port DEVICES-PER-PORT] + [--schemas-dir SCHEMAS-DIR] [--notification-file NOTIFICATION-FILE] + [--initial-config-xml-file INITIAL-CONFIG-XML-FILE] + [--starting-port STARTING-PORT] [--generate-config-connection-timeout GENERATE-CONFIG-CONNECTION-TIMEOUT] - [--generate-config-address GENERATE-CONFIG-ADDRESS] [--generate-configs-batch-size GENERATE-CONFIGS-BATCH-SIZE] - [--distribution-folder DISTRO-FOLDER] [--ssh {true,false}] [--exi {true,false}] [--debug {true,false}] - [--md-sal {true,false}] [--time-out TIME-OUT] [-ip IP] [--thread-pool-size THREAD-POOL-SIZE] [--rpc-config RPC-CONFIG] + [--generate-config-address GENERATE-CONFIG-ADDRESS] + [--generate-configs-batch-size GENERATE-CONFIGS-BATCH-SIZE] + [--distribution-folder DISTRO-FOLDER] [--ssh {true,false}] + [--exi {true,false}] [--debug {true,false}] [--md-sal {true,false}] + [--time-out TIME-OUT] [-ip IP] [--thread-pool-size THREAD-POOL-SIZE] + [--rpc-config RPC-CONFIG] netconf testtool @@ -162,26 +170,40 @@ Testtool help --async-requests {true,false} --thread-amount THREAD-AMOUNT The number of threads to use for configuring devices. - --throttle THROTTLE Maximum amount of async requests that can be open at a time, with mutltiple threads this gets divided among all threads - --auth AUTH AUTH Username and password for HTTP basic authentication in order username password. - --controller-destination CONTROLLER-DESTINATION - Ip address and port of controller. Must be in following format : if available it will be used for spawning - netconf connectors via topology configuration as a part of URI. Example (http:///restconf/config/network-topology:network-topology/topology/topology-netconf/node/)otherwise it will - just start simulated devices and skip the execution of PUT requests + --throttle THROTTLE Maximum amount of async requests that can be open at a time, with + mutltiple threads this gets divided among all threads + --controller-auth-username CONTROLLER-AUTH-USERNAME + Username for HTTP basic authentication to destination controller. + --controller-auth-password CONTROLLER-AUTH-PASSWORD + Password for HTTP basic authentication to destination controller. + --controller-ip CONTROLLER-IP + Ip of controller if available it will be used for spawning netconf + connectors via topology configuration as a part of URI(http://:/restconf/config/...) otherwise it will just start + simulated devices and skip the execution of PATCH requests + --controller-port CONTROLLER-PORT + Port of controller if available it will be used for spawning netconf + connectors via topology configuration as a part of URI(http://:/restconf/config/...) otherwise it will just start + simulated devices and skip the execution of PATCH requests --device-count DEVICES-COUNT - Number of simulated netconf devices to spin. This is the number of actual ports open for the devices. + Number of simulated netconf devices to spin. This is the number of actual + ports open for the devices. --devices-per-port DEVICES-PER-PORT - Amount of config files generated per port to spoof more devices than are actually running + Amount of config files generated per port to spoof more devices than are + actually running --schemas-dir SCHEMAS-DIR - Directory containing yang schemas to describe simulated devices. Some schemas e.g. netconf monitoring and inet types are - included by default + Directory containing yang schemas to describe simulated devices. Some + schemas e.g. netconf monitoring and inet types are included by default --notification-file NOTIFICATION-FILE - Xml file containing notifications that should be sent to clients after create subscription is called + Xml file containing notifications that should be sent to clients after + create subscription is called --initial-config-xml-file INITIAL-CONFIG-XML-FILE - Xml file containing initial simulatted configuration to be returned via get-config rpc + Xml file containing initial simulatted configuration to be returned via + get-config rpc --starting-port STARTING-PORT - First port for simulated device. Each other device will have previous+1 port number + First port for simulated device. Each other device will have previous+1 + port number --generate-config-connection-timeout GENERATE-CONFIG-CONNECTION-TIMEOUT Timeout to be generated in initial config files --generate-config-address GENERATE-CONFIG-ADDRESS @@ -194,14 +216,17 @@ Testtool help --exi {true,false} Whether to use exi to transport xml content --debug {true,false} Whether to use debug log level instead of INFO --md-sal {true,false} Whether to use md-sal datastore instead of default simulated datastore. - --time-out TIME-OUT the maximum time in seconds for executing each PUT request - -ip IP Ip address which will be used for creating a socket address.It can either be a machine name, such as java.sun.com, or a - textual representation of its IP address. + --time-out TIME-OUT the maximum time in seconds for executing each PATCH request + -ip IP Ip address which will be used for creating a socket address.It can either + be a machine name, such as java.sun.com, or a textual representation of + its IP address. --thread-pool-size THREAD-POOL-SIZE - The number of threads to keep in the pool, when creating a device simulator. Even if they are idle. + The number of threads to keep in the pool, when creating a device + simulator. Even if they are idle. --rpc-config RPC-CONFIG - Rpc config file. It can be used to define custom rpc behavior, or override the default one.Usable for testing buggy device - behavior. + Rpc config file. It can be used to define custom rpc behavior, or + override the default one.Usable for testing buggy device behavior. + Supported operations diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java index 208ee76c84..05957204b7 100644 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java +++ b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Execution.java @@ -7,6 +7,7 @@ */ package org.opendaylight.netconf.test.tool; +import com.google.common.collect.Lists; import java.io.IOException; import java.net.Authenticator; import java.net.PasswordAuthentication; @@ -17,50 +18,30 @@ import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.net.http.HttpResponse.BodyHandlers; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.concurrent.Callable; import java.util.concurrent.Semaphore; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class Execution implements Callable { +final class Execution implements Callable { + private static final Logger LOG = LoggerFactory.getLogger(Execution.class); + private static final String NETCONF_TOPOLOGY_DESTINATION = + "http://%s:%s/restconf/config/network-topology:network-topology/topology/topology-netconf"; - private final ArrayList payloads; private final HttpClient httpClient; - private static final Logger LOG = LoggerFactory.getLogger(Execution.class); - private final boolean invokeAsync; + private final String destination; + private final List openDevices; + private final TesttoolParameters params; private final Semaphore semaphore; - private final int throttle; - - static final class DestToPayload { - - private final String destination; - private final String payload; - - DestToPayload(final String destination, final String payload) { - this.destination = destination; - this.payload = payload; - } - - public String getDestination() { - return destination; - } - - public String getPayload() { - return payload; - } - } - public Execution(final TesttoolParameters params, final ArrayList payloads) { - this.invokeAsync = params.async; - this.throttle = params.throttle / params.threadAmount; - - if (params.async && params.threadAmount > 1) { - LOG.info("Throttling per thread: {}", this.throttle); - } - this.semaphore = new Semaphore(this.throttle); + private final int throttle; + private final boolean isAsync; - this.httpClient = HttpClient.newBuilder() + Execution(final List openDevices, final TesttoolParameters params) { + httpClient = HttpClient.newBuilder() .authenticator(new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { @@ -69,81 +50,87 @@ public class Execution implements Callable { } }) .build(); - this.payloads = new ArrayList<>(); - for (DestToPayload payload : payloads) { - HttpRequest request = HttpRequest.newBuilder(URI.create(payload.getDestination())) - .POST(BodyPublishers.ofString(payload.getPayload(), StandardCharsets.UTF_8)) - .header("Content-Type", "application/json") - .header("Accept", "application/json") - .build(); + destination = String.format(Locale.ROOT, NETCONF_TOPOLOGY_DESTINATION, + params.controllerIp, params.controllerPort); + this.openDevices = openDevices; + this.params = params; + + throttle = params.throttle / params.threadAmount; + isAsync = params.async; - this.payloads.add(request); + if (params.async && params.threadAmount > 1) { + LOG.info("Throttling per thread: {}", throttle); } + semaphore = new Semaphore(throttle); } - private void invokeSync() { - LOG.info("Begin sending sync requests"); - for (HttpRequest request : payloads) { - try { - HttpResponse response = httpClient.send(request, BodyHandlers.ofString()); - - if (response.statusCode() != 200 && response.statusCode() != 204) { - if (response.statusCode() == 409) { - LOG.warn("Request failed, status code: {} - one or more of the devices" - + " is already configured, skipping the whole batch", response.statusCode()); - } else { - LOG.warn("Status code: {}", response.statusCode()); - LOG.warn("url: {}", request.uri()); - LOG.warn("body: {}", response.body()); - } - } - } catch (InterruptedException | IOException e) { - LOG.warn("Failed to execute request", e); - } + @Override + public Void call() { + final List requests = prepareRequests(); + if (isAsync) { + this.sendAsync(requests); + } else { + this.sendSync(requests); } - LOG.info("End sending sync requests"); + return null; } - private void invokeAsync() { - LOG.info("Begin sending async requests"); + private List prepareRequests() { + final List> batches = Lists.partition(openDevices, params.generateConfigBatchSize); + return batches.stream() + .map(b -> PayloadCreator.createStringPayload(b, params)) + .map(this::prepareRequest) + .collect(Collectors.toList()); + } - for (final HttpRequest request : payloads) { + private void sendAsync(final List requests) { + LOG.info("Begin sending async requests"); + for (final HttpRequest request : requests) { try { semaphore.acquire(); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { LOG.warn("Semaphore acquire interrupted"); } httpClient.sendAsync(request, BodyHandlers.ofString()).whenComplete((response, error) -> { - if (response.statusCode() != 200 && response.statusCode() != 204) { - if (response.statusCode() == 409) { - LOG.warn("Request failed, status code: {} - one or more of the devices" - + " is already configured, skipping the whole batch", response.statusCode()); - } else { - LOG.warn("Request failed, status code: {}", response.statusCode()); - LOG.warn("request: {}", request); - } + if (response.statusCode() != 200) { + LOG.warn("Unexpected status code: {} for request to uri: {} with body: {}", + response.statusCode(), request.uri(), response.body()); } semaphore.release(); }); } LOG.info("Requests sent, waiting for responses"); - try { semaphore.acquire(this.throttle); - } catch (InterruptedException e) { + } catch (final InterruptedException e) { LOG.warn("Semaphore acquire interrupted"); } - LOG.info("Responses received, ending..."); } - @Override - public Void call() { - if (invokeAsync) { - this.invokeAsync(); - } else { - this.invokeSync(); + private void sendSync(final List requests) { + LOG.info("Begin sending sync requests"); + for (final HttpRequest request : requests) { + try { + final HttpResponse response = httpClient.send(request, BodyHandlers.ofString()); + if (response.statusCode() != 200) { + LOG.warn("Unexpected status code: {} for request to uri: {} with body: {}", + response.statusCode(), request.uri(), response.body()); + } + } catch (final InterruptedException | IOException e) { + LOG.error("Failed to execute request: {}", request, e); + throw new RuntimeException("Failed to execute request", e); + } } - return null; + LOG.info("End sending sync requests"); + } + + private HttpRequest prepareRequest(final String payload) { + LOG.info("Creating request to: {} with payload: {}", destination, payload); + return HttpRequest.newBuilder(URI.create(destination)) + .method("PATCH", BodyPublishers.ofString(payload, StandardCharsets.UTF_8)) + .header("Content-Type", "application/json") + .header("Accept", "application/json") + .build(); } } diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Main.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Main.java index e2d5a41bec..b5984a65fa 100644 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Main.java +++ b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/Main.java @@ -5,19 +5,21 @@ * terms of the Eclipse Public License v1.0 which accompanies this distribution, * and is available at http://www.eclipse.org/legal/epl-v10.html */ - package org.opendaylight.netconf.test.tool; import ch.qos.logback.classic.Level; import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.math.IntMath; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.ArrayList; +import java.math.RoundingMode; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.opendaylight.netconf.test.tool.config.Configuration; import org.opendaylight.netconf.test.tool.config.ConfigurationBuilder; import org.slf4j.Logger; @@ -28,7 +30,7 @@ public final class Main { private static final Logger LOG = LoggerFactory.getLogger(Main.class); private Main() { - + // hidden on purpose } @SuppressWarnings("checkstyle:IllegalCatch") @@ -51,20 +53,16 @@ public final class Main { } //if ODL controller ip is not set NETCONF devices will be started, but not registered at the controller if (params.controllerIp != null) { - final ArrayList> allThreadsPayloads = params - .getThreadsPayloads(openDevices); - final ArrayList executions = new ArrayList<>(); - for (ArrayList payloads : allThreadsPayloads) { - executions.add(new Execution(params, payloads)); - } + final List executionThreads = divideDevicesForThreads(openDevices, params); final ExecutorService executorService = Executors.newFixedThreadPool(params.threadAmount); final Stopwatch time = Stopwatch.createStarted(); - List> futures = executorService.invokeAll(executions, params.timeOut, TimeUnit.SECONDS); + final List> futures = executorService.invokeAll(executionThreads, + params.timeOut, TimeUnit.SECONDS); int threadNum = 0; - for (Future future : futures) { + for (final Future future : futures) { threadNum++; if (future.isCancelled()) { - LOG.info("{}. thread timed out.",threadNum); + LOG.info("{}. thread timed out.", threadNum); } else { try { future.get(); @@ -74,9 +72,9 @@ public final class Main { } } time.stop(); - LOG.info("Time spent with configuration of devices: {}.",time); + LOG.info("Time spent with configuration of devices: {}.", time); } - } catch (RuntimeException | InterruptedException e) { + } catch (final RuntimeException | InterruptedException e) { LOG.error("Unhandled exception", e); netconfDeviceSimulator.close(); System.exit(1); @@ -91,4 +89,12 @@ public final class Main { } } } + + private static List divideDevicesForThreads(final List openDevices, + final TesttoolParameters params) { + final int devicesPerThread = IntMath.divide(openDevices.size(), params.threadAmount, RoundingMode.UP); + return Lists.partition(openDevices, devicesPerThread).stream() + .map(t -> new Execution(t, params)) + .collect(Collectors.toList()); + } } diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/PayloadCreator.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/PayloadCreator.java new file mode 100644 index 0000000000..9bca8bba77 --- /dev/null +++ b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/PayloadCreator.java @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.netconf.test.tool; + +import static org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier; +import static org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates; + +import com.google.common.collect.ImmutableList; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.util.List; +import org.opendaylight.mdsal.binding.runtime.spi.BindingRuntimeHelpers; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netconf.node.topology.rev150114.$YangModuleInfoImpl; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netconf.node.topology.rev150114.NetconfNodeFields; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netconf.node.topology.rev150114.netconf.node.credentials.Credentials; +import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NetworkTopology; +import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.Topology; +import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node; +import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.Uint16; +import org.opendaylight.yangtools.yang.data.api.schema.ChoiceNode; +import org.opendaylight.yangtools.yang.data.api.schema.LeafNode; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeWriter; +import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactory; +import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactorySupplier; +import org.opendaylight.yangtools.yang.data.codec.gson.JSONNormalizedNodeStreamWriter; +import org.opendaylight.yangtools.yang.data.codec.gson.JsonWriterFactory; +import org.opendaylight.yangtools.yang.data.impl.schema.Builders; +import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext; +import org.opendaylight.yangtools.yang.model.api.SchemaPath; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class PayloadCreator { + private static final Logger LOG = LoggerFactory.getLogger(PayloadCreator.class); + + private static final EffectiveModelContext NETWORK_TOPOLOGY_SCHEMA_CONTEXT = BindingRuntimeHelpers + .createEffectiveModel(ImmutableList.of($YangModuleInfoImpl.getInstance())); + private static final JSONCodecFactory NETWORK_TOPOLOGY_JSON_CODEC_FACTORY = JSONCodecFactorySupplier + .DRAFT_LHOTKA_NETMOD_YANG_JSON_02.getShared(NETWORK_TOPOLOGY_SCHEMA_CONTEXT); + + private static final QName TOPOLOGY_ID_QNAME = QName.create(Topology.QNAME, "topology-id").intern(); + private static final QName NODE_ID_QNAME = QName.create(Node.QNAME, "node-id").intern(); + + private static final NodeIdentifier PORT_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME, "port")); + private static final NodeIdentifier HOST_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME,"host")); + private static final NodeIdentifier USERNAME_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME,"username")); + private static final NodeIdentifier PASSWORD_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME, "password")); + private static final NodeIdentifier CREDENTIALS_NODE_IDENTIFIER = NodeIdentifier.create(Credentials.QNAME); + private static final NodeIdentifier TCP_ONLY_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME, "tcp-only")); + private static final NodeIdentifier KEEPALIVE_DELAY_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME, "keepalive-delay")); + private static final NodeIdentifier SCHEMALESS_NODE_IDENTIFIER = NodeIdentifier + .create(QName.create(NetconfNodeFields.QNAME, "schemaless")); + private static final String DEFAULT_TOPOLOGY_ID = "topology-netconf"; + private static final String DEFAULT_NODE_PASSWORD = "admin"; + private static final String DEFAULT_NODE_USERNAME = "admin"; + + private static final boolean DEFAULT_NODE_SCHEMALESS = false; + private static final int DEFAULT_NODE_KEEPALIVE_DELAY = 0; + private static final int DEFAULT_REQUEST_PAYLOAD_INDENTATION = 2; + + private PayloadCreator() { + // hidden on purpose + } + + static String createStringPayload(final List devices, final TesttoolParameters parameters) { + return normalizedNodeToString(createNormalizedNodePayload(devices, parameters)); + } + + private static String normalizedNodeToString(final NormalizedNode node) { + final StringWriter writer = new StringWriter(); + final JsonWriter jsonWriter = JsonWriterFactory.createJsonWriter(writer, DEFAULT_REQUEST_PAYLOAD_INDENTATION); + final NormalizedNodeStreamWriter jsonStream = JSONNormalizedNodeStreamWriter.createExclusiveWriter( + NETWORK_TOPOLOGY_JSON_CODEC_FACTORY, SchemaPath.create(true, NetworkTopology.QNAME), + null, jsonWriter); + try (NormalizedNodeWriter nodeWriter = NormalizedNodeWriter.forStreamWriter(jsonStream)) { + nodeWriter.write(node); + } catch (final IOException e) { + LOG.error("Failed to serialize node: {} to JSON", node, e); + throw new RuntimeException("Failed to serialize node to JSON", e); + } + return writer.toString(); + } + + private static NormalizedNode createNormalizedNodePayload(final List devices, + final TesttoolParameters parameters) { + final var nodeBuilder = Builders.mapBuilder().withNodeIdentifier(NodeIdentifier.create(Node.QNAME)); + for (final Integer device : devices) { + nodeBuilder.withChild(Builders.mapEntryBuilder() + .withNodeIdentifier(NodeIdentifierWithPredicates.of(Node.QNAME, NODE_ID_QNAME, + createNodeID(device))) + .withChild(leafPort(device)) + .withChild(leafHost(parameters.generateConfigsAddress)) + .withChild(containerCredentials(DEFAULT_NODE_USERNAME, DEFAULT_NODE_PASSWORD)) + .withChild(leafTcpOnly(!parameters.ssh)) + .withChild(leafKeepaliveDelay(DEFAULT_NODE_KEEPALIVE_DELAY)) + .withChild(leafSchemaless(DEFAULT_NODE_SCHEMALESS)) + .build()); + } + + return Builders.mapBuilder() + .withNodeIdentifier(NodeIdentifier.create(Topology.QNAME)) + .withChild(Builders.mapEntryBuilder().withNodeIdentifier(NodeIdentifierWithPredicates + .of(Topology.QNAME, TOPOLOGY_ID_QNAME, DEFAULT_TOPOLOGY_ID)) + .withChild(nodeBuilder.build()) + .build()) + .build(); + } + + private static String createNodeID(final Integer port) { + return String.format("%d-sim-device", port); + } + + private static LeafNode leafPort(final int port) { + return Builders.leafBuilder() + .withNodeIdentifier(PORT_NODE_IDENTIFIER) + .withValue(Uint16.valueOf(port)) + .build(); + } + + private static LeafNode leafHost(final String host) { + return Builders.leafBuilder() + .withNodeIdentifier(HOST_NODE_IDENTIFIER) + .withValue(host) + .build(); + } + + private static ChoiceNode containerCredentials(final String username, final String password) { + return Builders.choiceBuilder().withNodeIdentifier(CREDENTIALS_NODE_IDENTIFIER) + .withChild(Builders.leafBuilder() + .withNodeIdentifier(USERNAME_NODE_IDENTIFIER) + .withValue(username) + .build()) + .withChild(Builders.leafBuilder() + .withNodeIdentifier(PASSWORD_NODE_IDENTIFIER) + .withValue(password) + .build()) + .build(); + } + + private static LeafNode leafTcpOnly(final Boolean tcpOnly) { + return Builders.leafBuilder() + .withNodeIdentifier(TCP_ONLY_NODE_IDENTIFIER) + .withValue(tcpOnly) + .build(); + } + + private static LeafNode leafKeepaliveDelay(final Integer keepaliveDelay) { + return Builders.leafBuilder() + .withNodeIdentifier(KEEPALIVE_DELAY_NODE_IDENTIFIER) + .withValue(keepaliveDelay) + .build(); + } + + private static LeafNode leafSchemaless(final Boolean schemaless) { + return Builders.leafBuilder() + .withNodeIdentifier(SCHEMALESS_NODE_IDENTIFIER) + .withValue(schemaless) + .build(); + } +} diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java index f7eaa7e4d7..6d32c24b14 100644 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java +++ b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java @@ -10,23 +10,18 @@ package org.opendaylight.netconf.test.tool; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import com.google.common.io.CharStreams; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.StringJoiner; import java.util.concurrent.TimeUnit; @@ -38,19 +33,12 @@ import net.sourceforge.argparse4j.inf.ArgumentParser; import net.sourceforge.argparse4j.inf.ArgumentParserException; import org.opendaylight.yangtools.yang.common.YangConstants; -@SuppressFBWarnings({"DM_EXIT", "DM_DEFAULT_ENCODING"}) -public class TesttoolParameters { - - private static final String HOST_KEY = "{HOST}"; - private static final String PORT_KEY = "{PORT}"; - private static final String TCP_ONLY = "{TCP_ONLY}"; - private static final String RESTCONF_NETCONF_TOPOLOGY_PATH_TEMPLATE = - "http://%s:%s/restconf/config/network-topology:network-topology/topology/topology-netconf/"; +@SuppressFBWarnings({"DM_EXIT"}) +public final class TesttoolParameters { private static final Pattern YANG_FILENAME_PATTERN = Pattern - .compile("(?.*)@(?\\d{4}-\\d{2}-\\d{2})\\.yang"); + .compile("(?.*)@(?\\d{4}-\\d{2}-\\d{2})\\.yang"); private static final Pattern REVISION_DATE_PATTERN = Pattern.compile("revision\\s+\"?(\\d{4}-\\d{2}-\\d{2})\"?"); - private static final String RESOURCE = "/config-template.json"; @Arg(dest = "async") public boolean async; @Arg(dest = "thread-amount") @@ -147,7 +135,7 @@ public class TesttoolParameters { .help("Ip of controller if available it will be used for spawning netconf connectors via topology" + " configuration as a part of" + " URI(http://:/restconf/config/...)" - + " otherwise it will just start simulated devices and skip the execution of PUT requests") + + " otherwise it will just start simulated devices and skip the execution of PATCH requests") .dest("controller-ip"); parser.addArgument("--controller-port") @@ -155,7 +143,7 @@ public class TesttoolParameters { .help("Port of controller if available it will be used for spawning netconf connectors via topology " + "configuration as a part of" + " URI(http://:/restconf/config/...) " - + "otherwise it will just start simulated devices and skip the execution of PUT requests") + + "otherwise it will just start simulated devices and skip the execution of PATCH requests") .dest("controller-port"); parser.addArgument("--device-count") @@ -241,7 +229,7 @@ public class TesttoolParameters { parser.addArgument("--time-out") .type(long.class) .setDefault(20) - .help("the maximum time in seconds for executing each PUT request") + .help("the maximum time in seconds for executing each PATCH request") .dest("time-out"); parser.addArgument("-ip") @@ -266,7 +254,7 @@ public class TesttoolParameters { return parser; } - public static TesttoolParameters parseArgs(final String[] args, final ArgumentParser parser) { + static TesttoolParameters parseArgs(final String[] args, final ArgumentParser parser) { final TesttoolParameters opt = new TesttoolParameters(); try { parser.parseArgs(args, opt); @@ -279,25 +267,6 @@ public class TesttoolParameters { return null; } - private static String modifyMessage(final StringBuilder payloadBuilder, final int payloadPosition, final int size) { - if (size == 1) { - return payloadBuilder.toString(); - } - - if (payloadPosition == 0) { - payloadBuilder.insert(payloadBuilder.toString().indexOf('{', 2), "["); - payloadBuilder.replace(payloadBuilder.length() - 1, payloadBuilder.length(), ","); - } else if (payloadPosition + 1 == size) { - payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1); - payloadBuilder.insert(payloadBuilder.toString().indexOf('}', 2) + 1, "]"); - } else { - payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1); - payloadBuilder.replace(payloadBuilder.length() - 2, payloadBuilder.length() - 1, ","); - payloadBuilder.deleteCharAt(payloadBuilder.toString().lastIndexOf('}')); - } - return payloadBuilder.toString(); - } - @SuppressWarnings("checkstyle:regexpSinglelineJava") void validate() { if (controllerIp != null) { @@ -312,7 +281,7 @@ public class TesttoolParameters { checkArgument(deviceCount > 0, "Device count has to be > 0"); checkArgument(startingPort > 1023, "Starting port has to be > 1023"); - checkArgument(devicesPerPort > 0, "Atleast one device per port needed"); + checkArgument(devicesPerPort > 0, "At least one device per port needed"); if (schemasDir != null) { checkArgument(schemasDir.exists(), "Schemas dir has to exist"); @@ -363,140 +332,6 @@ public class TesttoolParameters { return null; } - - public ArrayList> getThreadsPayloads(final List openDevices) { - final String editContentString; - try { - final InputStream stream = TesttoolParameters.class.getResourceAsStream(RESOURCE); - editContentString = CharStreams.toString(new InputStreamReader(stream, StandardCharsets.UTF_8)); - } catch (final IOException e) { - throw new IllegalArgumentException("Cannot read content of " + RESOURCE, e); - } - - int from; - int to; - Iterator iterator; - - final ArrayList> allThreadsPayloads = new ArrayList<>(); - if (generateConfigBatchSize > 1) { - - final int batchedRequests = openDevices.size() / generateConfigBatchSize; - final int batchedRequestsPerThread = batchedRequests / threadAmount; - final int leftoverBatchedRequests = batchedRequests % threadAmount; - final int leftoverRequests = openDevices.size() - batchedRequests * generateConfigBatchSize; - - //FIXME Move this to validate() and rename it to init() or create init() and move there. - //FIXME Make it field. - final String restconfNetconfTopologyPath = String.format(RESTCONF_NETCONF_TOPOLOGY_PATH_TEMPLATE, - controllerIp, controllerPort); - - for (int l = 0; l < threadAmount; l++) { - from = l * batchedRequests * batchedRequestsPerThread; - to = from + batchedRequests * batchedRequestsPerThread; - iterator = openDevices.subList(from, to).iterator(); - allThreadsPayloads.add(createBatchedPayloads(batchedRequestsPerThread, iterator, editContentString, - restconfNetconfTopologyPath)); - } - ArrayList payloads = null; - if (leftoverBatchedRequests > 0) { - from = threadAmount * batchedRequests * batchedRequestsPerThread; - to = from + batchedRequests * batchedRequestsPerThread; - iterator = openDevices.subList(from, to).iterator(); - payloads = createBatchedPayloads(leftoverBatchedRequests, iterator, editContentString, - restconfNetconfTopologyPath); - } - String payload = ""; - - for (int j = 0; j < leftoverRequests; j++) { - from = openDevices.size() - leftoverRequests; - to = openDevices.size(); - iterator = openDevices.subList(from, to).iterator(); - final StringBuilder payloadBuilder = new StringBuilder( - prepareMessage(iterator.next(), editContentString)); - payload += modifyMessage(payloadBuilder, j, leftoverRequests); - } - if (leftoverRequests > 0 || leftoverBatchedRequests > 0) { - - if (payloads != null) { - payloads.add(new Execution.DestToPayload(restconfNetconfTopologyPath, payload)); - } - allThreadsPayloads.add(payloads); - } - } else { - final int requestPerThreads = openDevices.size() / threadAmount; - final int leftoverRequests = openDevices.size() % threadAmount; - - for (int i = 0; i < threadAmount; i++) { - from = i * requestPerThreads; - to = from + requestPerThreads; - iterator = openDevices.subList(from, to).iterator(); - allThreadsPayloads.add(createPayloads(iterator, editContentString)); - } - - if (leftoverRequests > 0) { - from = threadAmount * requestPerThreads; - to = from + leftoverRequests; - iterator = openDevices.subList(from, to).iterator(); - allThreadsPayloads.add(createPayloads(iterator, editContentString)); - } - } - return allThreadsPayloads; - } - - private String prepareMessage(final int openDevice, final String editContentString) { - final StringBuilder messageBuilder = new StringBuilder(editContentString); - - if (editContentString.contains(HOST_KEY)) { - messageBuilder.replace(messageBuilder.indexOf(HOST_KEY), - messageBuilder.indexOf(HOST_KEY) + HOST_KEY.length(), - generateConfigsAddress); - } - if (editContentString.contains(PORT_KEY)) { - while (messageBuilder.indexOf(PORT_KEY) != -1) { - messageBuilder.replace(messageBuilder.indexOf(PORT_KEY), - messageBuilder.indexOf(PORT_KEY) + PORT_KEY.length(), - Integer.toString(openDevice)); - } - } - if (editContentString.contains(TCP_ONLY)) { - messageBuilder.replace(messageBuilder.indexOf(TCP_ONLY), - messageBuilder.indexOf(TCP_ONLY) + TCP_ONLY.length(), - Boolean.toString(!ssh)); - } - return messageBuilder.toString(); - } - - private ArrayList createPayloads(final Iterator openDevices, - final String editContentString) { - final ArrayList payloads = new ArrayList<>(); - - while (openDevices.hasNext()) { - //FIXME Move this to validate() and rename it to init() or create init() and move there. - //FIXME Make it field. - final String restconfNetconfTopologyPath = String.format(RESTCONF_NETCONF_TOPOLOGY_PATH_TEMPLATE, - controllerIp, controllerPort); - payloads.add(new Execution.DestToPayload( - restconfNetconfTopologyPath, prepareMessage(openDevices.next(), editContentString))); - } - return payloads; - } - - private ArrayList createBatchedPayloads(final int batchedRequestsCount, - final Iterator openDevices, final String editContentString, final String destination) { - final ArrayList payloads = new ArrayList<>(); - - for (int i = 0; i < batchedRequestsCount; i++) { - StringBuilder payload = new StringBuilder(); - for (int j = 0; j < generateConfigBatchSize; j++) { - final StringBuilder payloadBuilder = new StringBuilder( - prepareMessage(openDevices.next(), editContentString)); - payload.append(modifyMessage(payloadBuilder, j, generateConfigBatchSize)); - } - payloads.add(new Execution.DestToPayload(destination, payload.toString())); - } - return payloads; - } - @Override public String toString() { final List fields = Arrays.asList(this.getClass().getDeclaredFields()); diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Node.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Node.java deleted file mode 100644 index 76acc997ba..0000000000 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Node.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others. All rights reserved. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v1.0 which accompanies this distribution, - * and is available at http://www.eclipse.org/legal/epl-v10.html - */ -package org.opendaylight.netconf.test.tool.model; - -public class Node { - private String nodeId; - private String host; - private Integer port; - private String username; - private String password; - private Boolean tcpOnly; - private Integer keepaliveDelay; - private Boolean schemaless; - - public Node() { - - } - - public Node(final String nodeId, final String host, final Integer port, final String username, - final String password, final Boolean tcpOnly, final Integer keepaliveDelay, final Boolean schemaless) { - this.nodeId = nodeId; - this.host = host; - this.port = port; - this.username = username; - this.password = password; - this.tcpOnly = tcpOnly; - this.keepaliveDelay = keepaliveDelay; - this.schemaless = schemaless; - } - - public String getNodeId() { - return nodeId; - } - - public void setNodeId(final String nodeId) { - this.nodeId = nodeId; - } - - public String getUsername() { - return username; - } - - public void setUsername(final String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(final String password) { - this.password = password; - } - - public String getHost() { - return host; - } - - public void setHost(final String host) { - this.host = host; - } - - public Integer getKeepaliveDelay() { - return keepaliveDelay; - } - - public void setKeepaliveDelay(final Integer keepaliveDelay) { - this.keepaliveDelay = keepaliveDelay; - } - - public Integer getPort() { - return port; - } - - public void setPort(final Integer port) { - this.port = port; - } - - public Boolean getSchemaless() { - return schemaless; - } - - public void setSchemaless(final Boolean schemaless) { - this.schemaless = schemaless; - } - - public Boolean getTcpOnly() { - return tcpOnly; - } - - public void setTcpOnly(final Boolean tcpOnly) { - this.tcpOnly = tcpOnly; - } -} \ No newline at end of file diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Payload.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Payload.java deleted file mode 100644 index 2bcf7add94..0000000000 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Payload.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others. All rights reserved. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v1.0 which accompanies this distribution, - * and is available at http://www.eclipse.org/legal/epl-v10.html - */ -package org.opendaylight.netconf.test.tool.model; - -public class Payload { - private Topology topology; - - public Payload() { - - } - - public Payload(final Topology topology) { - this.topology = topology; - } - - public Topology getTopology() { - return topology; - } - - public void setTopology(Topology topology) { - this.topology = topology; - } -} diff --git a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Topology.java b/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Topology.java deleted file mode 100644 index 4fb1ee10ed..0000000000 --- a/netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/model/Topology.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 PANTHEON.tech, s.r.o. and others. All rights reserved. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v1.0 which accompanies this distribution, - * and is available at http://www.eclipse.org/legal/epl-v10.html - */ -package org.opendaylight.netconf.test.tool.model; - -import com.google.gson.annotations.SerializedName; -import java.util.ArrayList; -import java.util.List; - -public class Topology { - private String topologyId; - - @SerializedName("node") - private List nodeList = new ArrayList<>(); - - public Topology() { - - } - - public Topology(final String topologyId) { - this.topologyId = topologyId; - } - - public void addNode(final Node node) { - this.nodeList.add(node); - } - - public String getTopologyId() { - return topologyId; - } - - public void setTopologyId(String topologyId) { - this.topologyId = topologyId; - } - - public List getNodeList() { - return nodeList; - } - - public void setNodeList(List nodeList) { - this.nodeList = nodeList; - } -} diff --git a/netconf/tools/netconf-testtool/src/main/resources/config-template.json b/netconf/tools/netconf-testtool/src/main/resources/config-template.json deleted file mode 100644 index f107c23b40..0000000000 --- a/netconf/tools/netconf-testtool/src/main/resources/config-template.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "node": { - "node-id": "{PORT}-sim-device", - "host": "{HOST}", - "port": "{PORT}", - "username": "admin", - "password": "admin", - "tcp-only": "{TCP_ONLY}", - "keepalive-delay": "0" - } -} diff --git a/netconf/tools/netconf-testtool/src/main/resources/config-template.xml b/netconf/tools/netconf-testtool/src/main/resources/config-template.xml deleted file mode 100644 index 70287b2246..0000000000 --- a/netconf/tools/netconf-testtool/src/main/resources/config-template.xml +++ /dev/null @@ -1,9 +0,0 @@ - - {PORT}-sim-device - {HOST} - {PORT} - admin - admin - {TCP_ONLY} - 0 - \ No newline at end of file -- 2.36.6