netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/TesttoolParameters.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.test.tool;
  10
  11 import static com.google.common.base.Preconditions.checkArgument;
  12
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.io.CharStreams;
  15 import com.google.common.io.Files;
  16 import java.io.BufferedReader;
  17 import java.io.File;
  18 import java.io.FileReader;
  19 import java.io.IOException;
  20 import java.io.InputStream;
  21 import java.io.InputStreamReader;
  22 import java.nio.charset.StandardCharsets;
  23 import java.util.ArrayList;
  24 import java.util.Arrays;
  25 import java.util.Iterator;
  26 import java.util.List;
  27 import java.util.concurrent.TimeUnit;
  28 import java.util.regex.Matcher;
  29 import java.util.regex.Pattern;
  30 import net.sourceforge.argparse4j.ArgumentParsers;
  31 import net.sourceforge.argparse4j.annotation.Arg;
  32 import net.sourceforge.argparse4j.inf.ArgumentParser;
  33 import net.sourceforge.argparse4j.inf.ArgumentParserException;
  34
  35 public class TesttoolParameters {
  36
  37     private static final String HOST_KEY = "{HOST}";
  38     private static final String PORT_KEY = "{PORT}";
  39     private static final String TCP_ONLY = "{TCP_ONLY}";
  40     private static final String ADDRESS_PORT = "{ADDRESS:PORT}";
  41     private static final String dest = "http://{ADDRESS:PORT}/restconf/config/network-topology:network-topology/topology/topology-netconf/";
  42     private static final Pattern YANG_FILENAME_PATTERN = Pattern.compile("(?<name>.*)@(?<revision>\\d{4}-\\d{2}-\\d{2})\\.yang");
  43     private static final Pattern DATE_PATTERN = Pattern.compile("(\\d{4}-\\d{2}-\\d{2})");
  44
  45     private static final String RESOURCE = "/config-template.json";
  46     @Arg(dest = "edit-content")
  47     public File editContent;
  48     @Arg(dest = "async")
  49     public boolean async;
  50     @Arg(dest = "thread-amount")
  51     public int threadAmount;
  52     @Arg(dest = "throttle")
  53     public int throttle;
  54     @Arg(dest = "auth")
  55     public ArrayList<String> auth;
  56     @Arg(dest = "controller-destination")
  57     public String controllerDestination;
  58     @Arg(dest = "schemas-dir")
  59     public File schemasDir;
  60     @Arg(dest = "devices-count")
  61     public int deviceCount;
  62     @Arg(dest = "devices-per-port")
  63     public int devicesPerPort;
  64     @Arg(dest = "starting-port")
  65     public int startingPort;
  66     @Arg(dest = "generate-config-connection-timeout")
  67     public int generateConfigsTimeout;
  68     @Arg(dest = "generate-config-address")
  69     public String generateConfigsAddress;
  70     @Arg(dest = "distro-folder")
  71     public File distroFolder;
  72     @Arg(dest = "generate-configs-batch-size")
  73     public int generateConfigBatchSize;
  74     @Arg(dest = "ssh")
  75     public boolean ssh;
  76     @Arg(dest = "exi")
  77     public boolean exi;
  78     @Arg(dest = "debug")
  79     public boolean debug;
  80     @Arg(dest = "notification-file")
  81     public File notificationFile;
  82     @Arg(dest = "md-sal")
  83     public boolean mdSal;
  84     @Arg(dest = "initial-config-xml-file")
  85     public File initialConfigXMLFile;
  86     @Arg(dest = "time-out")
  87     public long timeOut;
  88     private InputStream stream;
  89
  90     @Arg(dest = "ip")
  91     public String ip;
  92
  93     @Arg(dest = "thread-pool-size")
  94     public int threadPoolSize;
  95
  96     static ArgumentParser getParser() {
  97         final ArgumentParser parser = ArgumentParsers.newArgumentParser("netconf testtool");
  98
  99         parser.description("netconf testtool");
 100
 101         parser.addArgument("--edit-content")
 102                 .type(String.class)
 103                 .dest("edit-content");
 104
 105         parser.addArgument("--async-requests")
 106                 .type(Boolean.class)
 107                 .setDefault(false)
 108                 .dest("async");
 109
 110         parser.addArgument("--thread-amount")
 111                 .type(Integer.class)
 112                 .setDefault(1)
 113                 .dest("thread-amount")
 114                 .help("The number of threads to use for configuring devices.");
 115
 116         parser.addArgument("--throttle")
 117                 .type(Integer.class)
 118                 .setDefault(5000)
 119                 .help("Maximum amount of async requests that can be open at a time, " +
 120                         "with mutltiple threads this gets divided among all threads")
 121                 .dest("throttle");
 122
 123         parser.addArgument("--auth")
 124                 .nargs(2)
 125                 .help("Username and password for HTTP basic authentication in order username password.")
 126                 .dest("auth");
 127
 128         parser.addArgument("--controller-destination")
 129                 .type(String.class)
 130                 .help("Ip address and port of controller. Must be in following format <ip>:<port> " +
 131                         "if available it will be used for spawning netconf connectors via topology configuration as " +
 132                         "a part of URI. Example (http://<controller destination>/restconf/config/network-topology:network-topology/topology/topology-netconf/node/<node-id>)" +
 133                         "otherwise it will just start simulated devices and skip the execution of PUT requests")
 134                 .dest("controller-destination");
 135
 136         parser.addArgument("--device-count")
 137                 .type(Integer.class)
 138                 .setDefault(1)
 139                 .help("Number of simulated netconf devices to spin. This is the number of actual ports open for the devices.")
 140                 .dest("devices-count");
 141
 142         parser.addArgument("--devices-per-port")
 143                 .type(Integer.class)
 144                 .setDefault(1)
 145                 .help("Amount of config files generated per port to spoof more devices then are actually running")
 146                 .dest("devices-per-port");
 147
 148         parser.addArgument("--schemas-dir")
 149                 .type(File.class)
 150                 .help("Directory containing yang schemas to describe simulated devices. Some schemas e.g. netconf monitoring and inet types are included by default")
 151                 .dest("schemas-dir");
 152
 153         parser.addArgument("--notification-file")
 154                 .type(File.class)
 155                 .help("Xml file containing notifications that should be sent to clients after create subscription is called")
 156                 .dest("notification-file");
 157
 158         parser.addArgument("--initial-config-xml-file")
 159                 .type(File.class)
 160                 .help("Xml file containing initial simulatted configuration to be returned via get-config rpc")
 161                 .dest("initial-config-xml-file");
 162
 163         parser.addArgument("--starting-port")
 164                 .type(Integer.class)
 165                 .setDefault(17830)
 166                 .help("First port for simulated device. Each other device will have previous+1 port number")
 167                 .dest("starting-port");
 168
 169         parser.addArgument("--generate-config-connection-timeout")
 170                 .type(Integer.class)
 171                 .setDefault((int) TimeUnit.MINUTES.toMillis(30))
 172                 .help("Timeout to be generated in initial config files")
 173                 .dest("generate-config-connection-timeout");
 174
 175         parser.addArgument("--generate-config-address")
 176                 .type(String.class)
 177                 .setDefault("127.0.0.1")
 178                 .help("Address to be placed in generated configs")
 179                 .dest("generate-config-address");
 180
 181         parser.addArgument("--generate-configs-batch-size")
 182                 .type(Integer.class)
 183                 .setDefault(1)
 184                 .help("Number of connector configs per generated file")
 185                 .dest("generate-configs-batch-size");
 186
 187         parser.addArgument("--distribution-folder")
 188                 .type(File.class)
 189                 .help("Directory where the karaf distribution for controller is located")
 190                 .dest("distro-folder");
 191
 192         parser.addArgument("--ssh")
 193                 .type(Boolean.class)
 194                 .setDefault(true)
 195                 .help("Whether to use ssh for transport or just pure tcp")
 196                 .dest("ssh");
 197
 198         parser.addArgument("--exi")
 199                 .type(Boolean.class)
 200                 .setDefault(true)
 201                 .help("Whether to use exi to transport xml content")
 202                 .dest("exi");
 203
 204         parser.addArgument("--debug")
 205                 .type(Boolean.class)
 206                 .setDefault(false)
 207                 .help("Whether to use debug log level instead of INFO")
 208                 .dest("debug");
 209
 210         parser.addArgument("--md-sal")
 211                 .type(Boolean.class)
 212                 .setDefault(false)
 213                 .help("Whether to use md-sal datastore instead of default simulated datastore.")
 214                 .dest("md-sal");
 215
 216         parser.addArgument("--time-out")
 217                 .type(long.class)
 218                 .setDefault(20)
 219                 .help("the maximum time in seconds for executing each PUT request")
 220                 .dest("time-out");
 221
 222         parser.addArgument("-ip")
 223                 .type(String.class)
 224                 .setDefault("0.0.0.0")
 225                 .help("Ip address which will be used for creating a socket address." +
 226                         "It can either be a machine name, such as " +
 227                         "java.sun.com, or a textual representation of its IP address.")
 228                 .dest("ip");
 229
 230         parser.addArgument("--thread-pool-size")
 231                 .type(Integer.class)
 232                 .setDefault(8)
 233                 .help("The number of threads to keep in the pool, when creating a device simulator. Even if they are idle.")
 234                 .dest("thread-pool-size");
 235
 236         return parser;
 237     }
 238
 239     public static TesttoolParameters parseArgs(final String[] args, final ArgumentParser parser) {
 240         final TesttoolParameters opt = new TesttoolParameters();
 241         try {
 242             parser.parseArgs(args, opt);
 243             return opt;
 244         } catch (final ArgumentParserException e) {
 245             parser.handleError(e);
 246         }
 247
 248         System.exit(1);
 249         return null;
 250     }
 251
 252     private static String modifyMessage(final StringBuilder payloadBuilder, final int payloadPosition, final int size) {
 253         if (size == 1) {
 254             return payloadBuilder.toString();
 255         }
 256
 257         if (payloadPosition == 0) {
 258             payloadBuilder.insert(payloadBuilder.toString().indexOf('{', 2), "[");
 259             payloadBuilder.replace(payloadBuilder.length() - 1, payloadBuilder.length(), ",");
 260         } else if (payloadPosition + 1 == size) {
 261             payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1);
 262             payloadBuilder.insert(payloadBuilder.toString().indexOf('}', 2) + 1, "]");
 263         } else {
 264             payloadBuilder.delete(0, payloadBuilder.toString().indexOf(':') + 1);
 265             payloadBuilder.replace(payloadBuilder.length() - 2, payloadBuilder.length() - 1, ",");
 266             payloadBuilder.deleteCharAt(payloadBuilder.toString().lastIndexOf('}'));
 267         }
 268         return payloadBuilder.toString();
 269     }
 270
 271     void validate() {
 272         if (editContent == null) {
 273             stream = TesttoolParameters.class.getResourceAsStream(RESOURCE);
 274         } else {
 275             Preconditions.checkArgument(!editContent.isDirectory(), "Edit content file is a dir");
 276             Preconditions.checkArgument(editContent.canRead(), "Edit content file is unreadable");
 277         }
 278
 279         if (controllerDestination != null) {
 280             Preconditions.checkArgument(controllerDestination.contains(":"), "Controller Destination needs to be in a following format <ip>:<port>");
 281             String[] parts = controllerDestination.split(Pattern.quote(":"));
 282             Preconditions.checkArgument(Integer.parseInt(parts[1]) > 0, "Port =< 0");
 283         }
 284
 285         checkArgument(deviceCount > 0, "Device count has to be > 0");
 286         checkArgument(startingPort > 1023, "Starting port has to be > 1023");
 287         checkArgument(devicesPerPort > 0, "Atleast one device per port needed");
 288
 289         if (schemasDir != null) {
 290             checkArgument(schemasDir.exists(), "Schemas dir has to exist");
 291             checkArgument(schemasDir.isDirectory(), "Schemas dir has to be a directory");
 292             checkArgument(schemasDir.canRead(), "Schemas dir has to be readable");
 293
 294             final List<File> files = Arrays.asList(schemasDir.listFiles());
 295             for (final File file : files) {
 296                 final Matcher matcher = YANG_FILENAME_PATTERN.matcher(file.getName());
 297                 if (!matcher.matches()) {
 298                     BufferedReader reader;
 299                     try {
 300                         reader = new BufferedReader(new FileReader(file));
 301                         String line = reader.readLine();
 302                         while (!DATE_PATTERN.matcher(line).find()) {
 303                             line = reader.readLine();
 304                         }
 305                         Matcher m = DATE_PATTERN.matcher(line);
 306
 307                         if (m.find()) {
 308                             String moduleName = file.getAbsolutePath();
 309                             if (file.getName().endsWith(".yang")) {
 310                                 moduleName = moduleName.substring(0, moduleName.length() - 5);
 311                             }
 312                             final String revision = m.group(1);
 313                             String correctName = moduleName + "@" + revision + ".yang";
 314                             File correctNameFile = new File(correctName);
 315                             file.renameTo(correctNameFile);
 316                         }
 317
 318                     } catch (IOException e) {
 319                         e.printStackTrace();
 320                     }
 321                 }
 322             }
 323         }
 324     }
 325
 326     public ArrayList<ArrayList<Execution.DestToPayload>> getThreadsPayloads(final List<Integer> openDevices) {
 327         final String editContentString;
 328         try {
 329             if (stream == null) {
 330                 editContentString = Files.toString(editContent, StandardCharsets.UTF_8);
 331             } else {
 332                 editContentString = CharStreams.toString(new InputStreamReader(stream, StandardCharsets.UTF_8));
 333             }
 334         } catch (final IOException e) {
 335             throw new IllegalArgumentException("Cannot read content of " + editContent);
 336         }
 337
 338         int from, to;
 339         Iterator<Integer> iterator;
 340
 341         final ArrayList<ArrayList<Execution.DestToPayload>> allThreadsPayloads = new ArrayList<>();
 342         if (generateConfigBatchSize > 1) {
 343
 344             final int batchedRequests = openDevices.size() / generateConfigBatchSize;
 345             final int batchedRequestsPerThread = batchedRequests / threadAmount;
 346             final int leftoverBatchedRequests = (batchedRequests) % threadAmount;
 347             final int leftoverRequests = openDevices.size() - (batchedRequests * generateConfigBatchSize);
 348
 349             final StringBuilder destBuilder = new StringBuilder(dest);
 350             destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT), destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(), controllerDestination);
 351
 352             for (int l = 0; l < threadAmount; l++) {
 353                 from = l * (batchedRequests * batchedRequestsPerThread);
 354                 to = from + (batchedRequests * batchedRequestsPerThread);
 355                 iterator = openDevices.subList(from, to).iterator();
 356                 allThreadsPayloads.add(createBatchedPayloads(batchedRequestsPerThread, iterator, editContentString, destBuilder.toString()));
 357             }
 358             ArrayList<Execution.DestToPayload> payloads = null;
 359             if (leftoverBatchedRequests > 0) {
 360                 from = threadAmount * (batchedRequests * batchedRequestsPerThread);
 361                 to = from + (batchedRequests * batchedRequestsPerThread);
 362                 iterator = openDevices.subList(from, to).iterator();
 363                 payloads = createBatchedPayloads(leftoverBatchedRequests, iterator, editContentString, destBuilder.toString());
 364             }
 365             String payload = "";
 366
 367             for (int j = 0; j < leftoverRequests; j++) {
 368                 from = openDevices.size() - leftoverRequests;
 369                 to = openDevices.size();
 370                 iterator = openDevices.subList(from, to).iterator();
 371                 final StringBuilder payloadBuilder = new StringBuilder(prepareMessage(iterator.next(), editContentString));
 372                 payload += modifyMessage(payloadBuilder, j, leftoverRequests);
 373             }
 374             if (leftoverRequests > 0 || leftoverBatchedRequests > 0) {
 375
 376                 if (payloads != null) {
 377                     payloads.add(new Execution.DestToPayload(destBuilder.toString(), payload));
 378                 }
 379                 allThreadsPayloads.add(payloads);
 380             }
 381         } else {
 382             final int requestPerThreads = openDevices.size() / threadAmount;
 383             final int leftoverRequests = openDevices.size() % threadAmount;
 384
 385             for (int i = 0; i < threadAmount; i++) {
 386                 from = i * requestPerThreads;
 387                 to = from + requestPerThreads;
 388                 iterator = openDevices.subList(from, to).iterator();
 389                 allThreadsPayloads.add(createPayloads(iterator, editContentString));
 390             }
 391
 392             if (leftoverRequests > 0) {
 393                 from = (threadAmount) * requestPerThreads;
 394                 to = from + leftoverRequests;
 395                 iterator = openDevices.subList(from, to).iterator();
 396                 allThreadsPayloads.add(createPayloads(iterator, editContentString));
 397             }
 398         }
 399         return allThreadsPayloads;
 400     }
 401
 402     private String prepareMessage(final int openDevice, final String editContentString) {
 403         StringBuilder messageBuilder = new StringBuilder(editContentString);
 404
 405         if (editContentString.contains(HOST_KEY)) {
 406             messageBuilder.replace(messageBuilder.indexOf(HOST_KEY), messageBuilder.indexOf(HOST_KEY) + HOST_KEY.length(), generateConfigsAddress);
 407         }
 408         if (editContentString.contains(PORT_KEY)) {
 409             while (messageBuilder.indexOf(PORT_KEY) != -1)
 410                 messageBuilder.replace(messageBuilder.indexOf(PORT_KEY), messageBuilder.indexOf(PORT_KEY) + PORT_KEY.length(), Integer.toString(openDevice));
 411         }
 412         if (editContentString.contains(TCP_ONLY)) {
 413             messageBuilder.replace(messageBuilder.indexOf(TCP_ONLY), messageBuilder.indexOf(TCP_ONLY) + TCP_ONLY.length(), Boolean.toString(!ssh));
 414         }
 415         return messageBuilder.toString();
 416     }
 417
 418     private ArrayList<Execution.DestToPayload> createPayloads(final Iterator<Integer> openDevices, final String editContentString) {
 419         final ArrayList<Execution.DestToPayload> payloads = new ArrayList<>();
 420
 421         while (openDevices.hasNext()) {
 422             final StringBuilder destBuilder = new StringBuilder(dest);
 423             destBuilder.replace(destBuilder.indexOf(ADDRESS_PORT), destBuilder.indexOf(ADDRESS_PORT) + ADDRESS_PORT.length(), controllerDestination);
 424             payloads.add(new Execution.DestToPayload(destBuilder.toString(), prepareMessage(openDevices.next(), editContentString)));
 425         }
 426         return payloads;
 427     }
 428
 429     private ArrayList<Execution.DestToPayload> createBatchedPayloads(final int batchedRequestsCount, final Iterator<Integer> openDevices, final String editContentString,
 430                                                                      final String destination) {
 431         final ArrayList<Execution.DestToPayload> payloads = new ArrayList<>();
 432
 433         for (int i = 0; i < batchedRequestsCount; i++) {
 434             String payload = "";
 435             for (int j = 0; j < generateConfigBatchSize; j++) {
 436                 final StringBuilder payloadBuilder = new StringBuilder(prepareMessage(openDevices.next(), editContentString));
 437                 payload += modifyMessage(payloadBuilder, j, generateConfigBatchSize);
 438             }
 439             payloads.add(new Execution.DestToPayload(destination, payload));
 440         }
 441         return payloads;
 442     }
 443
 444     //TODO This may be more scalable enumerating parameters via reflection
 445     @Override
 446     public String toString() {
 447         StringBuffer params = new StringBuffer("TesttoolParameters{");
 448         params.append("edit-content='").append(editContent).append('\'');
 449         params.append(", async='").append(async).append('\'');
 450         params.append(", thread-amount='").append(threadAmount).append('\'');
 451         params.append(", throttle='").append(throttle).append('\'');
 452         params.append(", auth='").append(auth).append('\'');
 453         params.append(", controller-destination='").append(controllerDestination).append('\'');
 454         params.append(", schemas-dir='").append(schemasDir).append('\'');
 455         params.append(", devices-count='").append(deviceCount).append('\'');
 456         params.append(", devices-per-port='").append(devicesPerPort).append('\'');
 457         params.append(", starting-port='").append(startingPort).append('\'');
 458         params.append(", generate-config-connection-timeout='").append(generateConfigsTimeout).append('\'');
 459         params.append(", generate-config-address='").append(generateConfigsAddress).append('\'');
 460         params.append(", distro-folder='").append(distroFolder).append('\'');
 461         params.append(", generate-configs-batch-size='").append(generateConfigBatchSize).append('\'');
 462         params.append(", ssh='").append(ssh).append('\'');
 463         params.append(", exi='").append(exi).append('\'');
 464         params.append(", debug='").append(debug).append('\'');
 465         params.append(", notification-file='").append(notificationFile).append('\'');
 466         params.append(", md-sal='").append(mdSal).append('\'');
 467         params.append(", initial-config-xml-file='").append(initialConfigXMLFile).append('\'');
 468         params.append(", time-out='").append(timeOut).append('\'');
 469         params.append('}');
 470
 471         return params.toString();
 472     }
 473 }