netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.test.tool;
  10
  11 import static com.google.common.base.Preconditions.checkArgument;
  12
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.io.CharStreams;
  15 import com.google.common.io.Files;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.io.BufferedReader;
  18 import java.io.File;
  19 import java.io.FileReader;
  20 import java.io.IOException;
  21 import java.io.InputStream;
  22 import java.io.InputStreamReader;
  23 import java.lang.reflect.Field;
  24 import java.nio.charset.StandardCharsets;
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.Collections;
  28 import java.util.Iterator;
  29 import java.util.List;
  30 import java.util.Objects;
  31 import java.util.StringJoiner;
  32 import java.util.concurrent.TimeUnit;
  33 import java.util.regex.Matcher;
  34 import java.util.regex.Pattern;
  35 import net.sourceforge.argparse4j.ArgumentParsers;
  36 import net.sourceforge.argparse4j.annotation.Arg;
  37 import net.sourceforge.argparse4j.inf.ArgumentParser;
  38 import net.sourceforge.argparse4j.inf.ArgumentParserException;
  39
  40 @SuppressFBWarnings({"DM_EXIT", "DM_DEFAULT_ENCODING"})
  41 public class TesttoolParameters {
  42
  43     private static final String HOST_KEY = "{HOST}";
  44     private static final String PORT_KEY = "{PORT}";
  45     private static final String TCP_ONLY = "{TCP_ONLY}";
  46     private static final String ADDRESS_PORT = "{ADDRESS:PORT}";
  47     private static final String DEST =
  48         "http://{ADDRESS:PORT}/restconf/config/network-topology:network-topology/topology/topology-netconf/";
  49     private static final Pattern YANG_FILENAME_PATTERN = Pattern
  50         .compile("(?<name>.*)@(?<revision>\\d{4}-\\d{2}-\\d{2})\\.yang");
  51     private static final Pattern REVISION_DATE_PATTERN = Pattern.compile("revision\\s+\"?(\\d{4}-\\d{2}-\\d{2})\"?");
  52
  53     private static final String RESOURCE = "/config-template.json";
  54     @Arg(dest = "edit-content")
  55     public File editContent;
  56     @Arg(dest = "async")
  57     public boolean async;
  58     @Arg(dest = "thread-amount")
  59     public int threadAmount;
  60     @Arg(dest = "throttle")
  61     public int throttle;
  62     @Arg(dest = "auth")
  63     public ArrayList<String> auth;
  64     @Arg(dest = "controller-destination")
  65     public String controllerDestination;
  66     @Arg(dest = "schemas-dir")
  67     public File schemasDir;
  68     @Arg(dest = "devices-count")
  69     public int deviceCount;
  70     @Arg(dest = "devices-per-port")
  71     public int devicesPerPort;
  72     @Arg(dest = "starting-port")
  73     public int startingPort;
  74     @Arg(dest = "generate-config-connection-timeout")
  75     public int generateConfigsTimeout;
  76     @Arg(dest = "generate-config-address")
  77     public String generateConfigsAddress;
  78     @Arg(dest = "distro-folder")
  79     public File distroFolder;
  80     @Arg(dest = "generate-configs-batch-size")
  81     public int generateConfigBatchSize;
  82     @Arg(dest = "ssh")
  83     public boolean ssh;
  84     @Arg(dest = "exi")
  85     public boolean exi = true;
  86     @Arg(dest = "debug")
  87     public boolean debug;
  88     @Arg(dest = "notification-file")
  89     public File notificationFile;
  90     @Arg(dest = "md-sal")
  91     public boolean mdSal;
  92     @Arg(dest = "initial-config-xml-file")
  93     public File initialConfigXMLFile;
  94     @Arg(dest = "time-out")
  95     public long timeOut;
  96     private InputStream stream;
  97     @Arg(dest = "ip")
  98     public String ip;
  99     @Arg(dest = "thread-pool-size")
 100     public int threadPoolSize;
 101     @Arg(dest = "rpc-config")
 102     public File rpcConfig;
 103
 104     @SuppressWarnings("checkstyle:lineLength")
 105     static ArgumentParser getParser() {
 106         final ArgumentParser parser = ArgumentParsers.newArgumentParser("netconf testtool");
 107
 108         parser.description("netconf testtool");
 109
 110         parser.addArgument("--edit-content")
 111                 .type(String.class)
 112                 .dest("edit-content");
 113
 114         parser.addArgument("--async-requests")
 115                 .type(Boolean.class)
 116                 .setDefault(Boolean.FALSE)
 117                 .dest("async");
 118
 119         parser.addArgument("--thread-amount")
 120                 .type(Integer.class)
 121                 .setDefault(1)
 122                 .dest("thread-amount")
 123                 .help("The number of threads to use for configuring devices.");
 124
 125         parser.addArgument("--throttle")
 126                 .type(Integer.class)
 127                 .setDefault(5000)
 128                 .help("Maximum amount of async requests that can be open at a time, "
 129                         + "with mutltiple threads this gets divided among all threads")
 130                 .dest("throttle");
 131
 132         parser.addArgument("--auth")
 133                 .nargs(2)
 134                 .help("Username and password for HTTP basic authentication in order username password.")
 135                 .dest("auth");
 136
 137         parser.addArgument("--controller-destination")
 138                 .type(String.class)
 139                 .help("Ip address and port of controller. Must be in following format <ip>:<port> "
 140                         + "if available it will be used for spawning netconf connectors via topology configuration as "
 141                         + "a part of URI. Example (http://<controller destination>/restconf/config/network-topology:network-topology/topology/topology-netconf/node/<node-id>)"
 142                         + "otherwise it will just start simulated devices and skip the execution of PUT requests")
 143                 .dest("controller-destination");
 144
 145         parser.addArgument("--device-count")
 146                 .type(Integer.class)
 147                 .setDefault(1)
 148                 .help("Number of simulated netconf devices to spin. This is the number of actual ports open for the devices.")
 149                 .dest("devices-count");
 150
 151         parser.addArgument("--devices-per-port")
 152                 .type(Integer.class)
 153                 .setDefault(1)
 154                 .help("Amount of config files generated per port to spoof more devices than are actually running")
 155                 .dest("devices-per-port");
 156
 157         parser.addArgument("--schemas-dir")
 158                 .type(File.class)
 159                 .help("Directory containing yang schemas to describe simulated devices. Some schemas e.g. netconf monitoring and inet types are included by default")
 160                 .dest("schemas-dir");
 161
 162         parser.addArgument("--notification-file")
 163                 .type(File.class)
 164                 .help("Xml file containing notifications that should be sent to clients after create subscription is called")
 165                 .dest("notification-file");
 166
 167         parser.addArgument("--initial-config-xml-file")
 168                 .type(File.class)
 169                 .help("Xml file containing initial simulatted configuration to be returned via get-config rpc")
 170                 .dest("initial-config-xml-file");
 171
 172         parser.addArgument("--starting-port")
 173                 .type(Integer.class)
 174                 .setDefault(17830)
 175                 .help("First port for simulated device. Each other device will have previous+1 port number")
 176                 .dest("starting-port");
 177
 178         parser.addArgument("--generate-config-connection-timeout")
 179                 .type(Integer.class)
 180                 .setDefault((int) TimeUnit.MINUTES.toMillis(30))
 181                 .help("Timeout to be generated in initial config files")
 182                 .dest("generate-config-connection-timeout");
 183
 184         parser.addArgument("--generate-config-address")
 185                 .type(String.class)
 186                 .setDefault("127.0.0.1")
 187                 .help("Address to be placed in generated configs")
 188                 .dest("generate-config-address");
 189
 190         parser.addArgument("--generate-configs-batch-size")
 191                 .type(Integer.class)
 192                 .setDefault(1)
 193                 .help("Number of connector configs per generated file")
 194                 .dest("generate-configs-batch-size");
 195
 196         parser.addArgument("--distribution-folder")
 197                 .type(File.class)
 198                 .help("Directory where the karaf distribution for controller is located")
 199                 .dest("distro-folder");
 200
 201         parser.addArgument("--ssh")
 202                 .type(Boolean.class)
 203                 .setDefault(Boolean.TRUE)
 204                 .help("Whether to use ssh for transport or just pure tcp")
 205                 .dest("ssh");
 206
 207         parser.addArgument("--exi")
 208                 .type(Boolean.class)
 209                 .setDefault(Boolean.TRUE)
 210                 .help("Whether to use exi to transport xml content")
 211                 .dest("exi");
 212
 213         parser.addArgument("--debug")
 214                 .type(Boolean.class)
 215                 .setDefault(Boolean.FALSE)
 216                 .help("Whether to use debug log level instead of INFO")
 217                 .dest("debug");
 218
 219         parser.addArgument("--md-sal")
 220                 .type(Boolean.class)
 221                 .setDefault(Boolean.FALSE)
 222                 .help("Whether to use md-sal datastore instead of default simulated datastore.")
 223                 .dest("md-sal");
 224
 225         parser.addArgument("--time-out")
 226                 .type(long.class)
 227                 .setDefault(20)
 228                 .help("the maximum time in seconds for executing each PUT request")
 229                 .dest("time-out");
 230
 231         parser.addArgument("-ip")
 232                 .type(String.class)
 233                 .setDefault("0.0.0.0")
 234                 .help("Ip address which will be used for creating a socket address."
 235                         + "It can either be a machine name, such as "
 236                         + "java.sun.com, or a textual representation of its IP address.")
 237                 .dest("ip");
 238
 239         parser.addArgument("--thread-pool-size")
 240                 .type(Integer.class)
 241                 .setDefault(8)
 242                 .help("The number of threads to keep in the pool, when creating a device simulator. Even if they are idle.")
 243                 .dest("thread-pool-size");
 244         parser.addArgument("--rpc-config")
 245                 .type(File.class)
 246                 .help("Rpc config file. It can be used to define custom rpc behavior, or override the default one."
 247                     + "Usable for testing buggy device behavior.")
 248                 .dest("rpc-config");
 249
 250         return parser;
 251     }
 252
 253     public static TesttoolParameters parseArgs(final String[] args, final ArgumentParser parser) {
 254         final TesttoolParameters opt = new TesttoolParameters();
 255         try {
 256             parser.parseArgs(args, opt);
 257             return opt;
 258         } catch (final ArgumentParserException e) {
 259             parser.handleError(e);
 260         }
 261
 262         System.exit(1);
 263         return null;
 264     }
 265
 266     private static String modifyMessage(final StringBuilder payloadBuilder, final int payloadPosition, final int size) {
 267         if (size == 1) {
 268             return payloadBuilder.toString();
 269         }
 270
 271         if (payloadPosition == 0) {
 272             payloadBuilder.insert(payloadBuilder.toString().indexOf('{', 2), "[");
 273             payloadBuilder.replace(payloadBuilder.length() - 1, payloadBuilder.length(), ",");
 274         } else if (payloadPosition + 1 == size) {
 275             payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1);
 276             payloadBuilder.insert(payloadBuilder.toString().indexOf('}', 2) + 1, "]");
 277         } else {
 278             payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1);
 279             payloadBuilder.replace(payloadBuilder.length() - 2, payloadBuilder.length() - 1, ",");
 280             payloadBuilder.deleteCharAt(payloadBuilder.toString().lastIndexOf('}'));
 281         }
 282         return payloadBuilder.toString();
 283     }
 284
 285     @SuppressWarnings("checkstyle:regexpSinglelineJava")
 286     void validate() {
 287         if (editContent == null) {
 288             stream = TesttoolParameters.class.getResourceAsStream(RESOURCE);
 289         } else {
 290             Preconditions.checkArgument(!editContent.isDirectory(), "Edit content file is a dir");
 291             Preconditions.checkArgument(editContent.canRead(), "Edit content file is unreadable");
 292         }
 293
 294         if (controllerDestination != null) {
 295             Preconditions.checkArgument(controllerDestination.contains(":"),
 296                 "Controller Destination needs to be in a following format <ip>:<port>");
 297             final String[] parts = controllerDestination.split(Pattern.quote(":"));
 298             Preconditions.checkArgument(Integer.parseInt(parts[1]) > 0, "Port =< 0");
 299         }
 300
 301         checkArgument(deviceCount > 0, "Device count has to be > 0");
 302         checkArgument(startingPort > 1023, "Starting port has to be > 1023");
 303         checkArgument(devicesPerPort > 0, "Atleast one device per port needed");
 304
 305         if (schemasDir != null) {
 306             checkArgument(schemasDir.exists(), "Schemas dir has to exist");
 307             checkArgument(schemasDir.isDirectory(), "Schemas dir has to be a directory");
 308             checkArgument(schemasDir.canRead(), "Schemas dir has to be readable");
 309
 310             final File[] filesArray = schemasDir.listFiles();
 311             final List<File> files = filesArray != null ? Arrays.asList(filesArray) : Collections.emptyList();
 312             for (final File file : files) {
 313                 final Matcher matcher = YANG_FILENAME_PATTERN.matcher(file.getName());
 314                 if (!matcher.matches()) {
 315                     try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
 316                         String line = reader.readLine();
 317                         while (Objects.nonNull(line) && !REVISION_DATE_PATTERN.matcher(line).find()) {
 318                             line = reader.readLine();
 319                         }
 320                         if (Objects.nonNull(line)) {
 321                             final Matcher m = REVISION_DATE_PATTERN.matcher(line);
 322                             Preconditions.checkState(m.find());
 323                             String moduleName = file.getAbsolutePath();
 324                             if (file.getName().endsWith(".yang")) {
 325                                 moduleName = moduleName.substring(0, moduleName.length() - 5);
 326                             }
 327                             final String revision = m.group(1);
 328                             final String correctName = moduleName + "@" + revision + ".yang";
 329                             final File correctNameFile = new File(correctName);
 330                             if (!file.renameTo(correctNameFile)) {
 331                                 throw new IllegalStateException("Failed to rename '%s'." + file);
 332                             }
 333                         }
 334                     } catch (final IOException e) {
 335                         // print error to console (test tool is running from console)
 336                         e.printStackTrace();
 337                     }
 338                 }
 339             }
 340         }
 341         if (rpcConfig != null) {
 342             checkArgument(rpcConfig.exists(), "Rpc config file has to exist");
 343             checkArgument(!rpcConfig.isDirectory(), "Rpc config file can't be a directory");
 344             checkArgument(rpcConfig.canRead(), "Rpc config file to be readable");
 345         }
 346     }
 347
 348     public ArrayList<ArrayList<Execution.DestToPayload>> getThreadsPayloads(final List<Integer> openDevices) {
 349         final String editContentString;
 350         try {
 351             if (stream == null) {
 352                 editContentString = Files.asCharSource(editContent, StandardCharsets.UTF_8).read();
 353             } else {
 354                 editContentString = CharStreams.toString(new InputStreamReader(stream, StandardCharsets.UTF_8));
 355             }
 356         } catch (final IOException e) {
 357             throw new IllegalArgumentException("Cannot read content of " + editContent, e);
 358         }
 359
 360         int from;
 361         int to;
 362         Iterator<Integer> iterator;
 363
 364         final ArrayList<ArrayList<Execution.DestToPayload>> allThreadsPayloads = new ArrayList<>();
 365         if (generateConfigBatchSize > 1) {
 366
 367             final int batchedRequests = openDevices.size() / generateConfigBatchSize;
 368             final int batchedRequestsPerThread = batchedRequests / threadAmount;
 369             final int leftoverBatchedRequests = batchedRequests % threadAmount;
 370             final int leftoverRequests = openDevices.size() - batchedRequests * generateConfigBatchSize;
 371
 372             final StringBuilder destBuilder = new StringBuilder(DEST);
 373             destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT),
 374                 destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(),
 375                 controllerDestination);
 376
 377             for (int l = 0; l < threadAmount; l++) {
 378                 from = l * batchedRequests * batchedRequestsPerThread;
 379                 to = from + batchedRequests * batchedRequestsPerThread;
 380                 iterator = openDevices.subList(from, to).iterator();
 381                 allThreadsPayloads.add(createBatchedPayloads(batchedRequestsPerThread, iterator, editContentString,
 382                     destBuilder.toString()));
 383             }
 384             ArrayList<Execution.DestToPayload> payloads = null;
 385             if (leftoverBatchedRequests > 0) {
 386                 from = threadAmount * batchedRequests * batchedRequestsPerThread;
 387                 to = from + batchedRequests * batchedRequestsPerThread;
 388                 iterator = openDevices.subList(from, to).iterator();
 389                 payloads = createBatchedPayloads(leftoverBatchedRequests, iterator, editContentString,
 390                     destBuilder.toString());
 391             }
 392             String payload = "";
 393
 394             for (int j = 0; j < leftoverRequests; j++) {
 395                 from = openDevices.size() - leftoverRequests;
 396                 to = openDevices.size();
 397                 iterator = openDevices.subList(from, to).iterator();
 398                 final StringBuilder payloadBuilder = new StringBuilder(
 399                     prepareMessage(iterator.next(), editContentString));
 400                 payload += modifyMessage(payloadBuilder, j, leftoverRequests);
 401             }
 402             if (leftoverRequests > 0 || leftoverBatchedRequests > 0) {
 403
 404                 if (payloads != null) {
 405                     payloads.add(new Execution.DestToPayload(destBuilder.toString(), payload));
 406                 }
 407                 allThreadsPayloads.add(payloads);
 408             }
 409         } else {
 410             final int requestPerThreads = openDevices.size() / threadAmount;
 411             final int leftoverRequests = openDevices.size() % threadAmount;
 412
 413             for (int i = 0; i < threadAmount; i++) {
 414                 from = i * requestPerThreads;
 415                 to = from + requestPerThreads;
 416                 iterator = openDevices.subList(from, to).iterator();
 417                 allThreadsPayloads.add(createPayloads(iterator, editContentString));
 418             }
 419
 420             if (leftoverRequests > 0) {
 421                 from = threadAmount * requestPerThreads;
 422                 to = from + leftoverRequests;
 423                 iterator = openDevices.subList(from, to).iterator();
 424                 allThreadsPayloads.add(createPayloads(iterator, editContentString));
 425             }
 426         }
 427         return allThreadsPayloads;
 428     }
 429
 430     private String prepareMessage(final int openDevice, final String editContentString) {
 431         final StringBuilder messageBuilder = new StringBuilder(editContentString);
 432
 433         if (editContentString.contains(HOST_KEY)) {
 434             messageBuilder.replace(messageBuilder.indexOf(HOST_KEY),
 435                 messageBuilder.indexOf(HOST_KEY) + HOST_KEY.length(),
 436                 generateConfigsAddress);
 437         }
 438         if (editContentString.contains(PORT_KEY)) {
 439             while (messageBuilder.indexOf(PORT_KEY) != -1) {
 440                 messageBuilder.replace(messageBuilder.indexOf(PORT_KEY),
 441                     messageBuilder.indexOf(PORT_KEY) + PORT_KEY.length(),
 442                     Integer.toString(openDevice));
 443             }
 444         }
 445         if (editContentString.contains(TCP_ONLY)) {
 446             messageBuilder.replace(messageBuilder.indexOf(TCP_ONLY),
 447                 messageBuilder.indexOf(TCP_ONLY) + TCP_ONLY.length(),
 448                 Boolean.toString(!ssh));
 449         }
 450         return messageBuilder.toString();
 451     }
 452
 453     private ArrayList<Execution.DestToPayload> createPayloads(final Iterator<Integer> openDevices,
 454                                                               final String editContentString) {
 455         final ArrayList<Execution.DestToPayload> payloads = new ArrayList<>();
 456
 457         while (openDevices.hasNext()) {
 458             final StringBuilder destBuilder = new StringBuilder(DEST);
 459             destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT),
 460                 destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(), controllerDestination);
 461             payloads.add(new Execution.DestToPayload(
 462                 destBuilder.toString(), prepareMessage(openDevices.next(), editContentString)));
 463         }
 464         return payloads;
 465     }
 466
 467     private ArrayList<Execution.DestToPayload> createBatchedPayloads(final int batchedRequestsCount,
 468             final Iterator<Integer> openDevices, final String editContentString, final String destination) {
 469         final ArrayList<Execution.DestToPayload> payloads = new ArrayList<>();
 470
 471         for (int i = 0; i < batchedRequestsCount; i++) {
 472             StringBuilder payload = new StringBuilder();
 473             for (int j = 0; j < generateConfigBatchSize; j++) {
 474                 final StringBuilder payloadBuilder = new StringBuilder(
 475                     prepareMessage(openDevices.next(), editContentString));
 476                 payload.append(modifyMessage(payloadBuilder, j, generateConfigBatchSize));
 477             }
 478             payloads.add(new Execution.DestToPayload(destination, payload.toString()));
 479         }
 480         return payloads;
 481     }
 482
 483     @Override
 484     public String toString() {
 485         final List<Field> fields = Arrays.asList(this.getClass().getDeclaredFields());
 486         final StringJoiner joiner = new StringJoiner(", \n", "TesttoolParameters{", "}\n");
 487         fields.stream()
 488                 .filter(field -> field.getAnnotation(Arg.class) != null)
 489                 .map(this::getFieldString)
 490                 .forEach(joiner::add);
 491         return joiner.toString();
 492     }
 493
 494     private String getFieldString(final Field field) {
 495         try {
 496             return field.getName() + "='" + field.get(this) + "'";
 497         } catch (final IllegalAccessException e) {
 498             return field.getName() + "= UNKNOWN";
 499         }
 500     }
 501 }