netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/ScaleUtil.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.test.tool;
  10
  11 import ch.qos.logback.classic.Level;
  12 import com.google.common.base.Charsets;
  13 import com.google.common.base.Stopwatch;
  14 import com.google.common.io.CharStreams;
  15 import com.ning.http.client.AsyncHttpClient;
  16 import com.ning.http.client.AsyncHttpClientConfig.Builder;
  17 import com.ning.http.client.Request;
  18 import com.ning.http.client.Response;
  19 import java.io.BufferedInputStream;
  20 import java.io.BufferedReader;
  21 import java.io.File;
  22 import java.io.IOException;
  23 import java.io.InputStreamReader;
  24 import java.net.ConnectException;
  25 import java.util.List;
  26 import java.util.concurrent.Callable;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.ScheduledExecutorService;
  29 import java.util.concurrent.ScheduledFuture;
  30 import java.util.concurrent.ScheduledThreadPoolExecutor;
  31 import java.util.concurrent.Semaphore;
  32 import java.util.concurrent.TimeUnit;
  33 import java.util.regex.Matcher;
  34 import java.util.regex.Pattern;
  35 import net.sourceforge.argparse4j.inf.ArgumentParser;
  36 import net.sourceforge.argparse4j.inf.ArgumentParserException;
  37 import org.opendaylight.netconf.test.tool.Main.Params;
  38 import org.slf4j.Logger;
  39 import org.slf4j.LoggerFactory;
  40
  41 public class ScaleUtil {
  42
  43     private static final Logger RESULTS_LOG = LoggerFactory.getLogger("results");
  44     private static final ScheduledExecutorService executor = new LoggingWrapperExecutor(4);
  45
  46     private static final int deviceStep = 1000;
  47     private static final long retryDelay = 10l;
  48     private static final long timeout = 20l;
  49
  50     private static final Stopwatch stopwatch = Stopwatch.createUnstarted();
  51
  52     private static ScheduledFuture timeoutGuardFuture;
  53     private static ch.qos.logback.classic.Logger root;
  54     private static final Semaphore semaphore = new Semaphore(0);
  55
  56     public static void main(final String[] args) {
  57         final Params params = parseArgs(args, Params.getParser());
  58
  59         root = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
  60         root.setLevel(params.debug ? Level.DEBUG : Level.INFO);
  61
  62         // cleanup at the start in case controller was already running
  63         final Runtime runtime = Runtime.getRuntime();
  64         cleanup(runtime, params);
  65
  66         while (true) {
  67             root.warn("Starting scale test with {} devices", params.deviceCount);
  68             timeoutGuardFuture = executor.schedule(new TimeoutGuard(), timeout, TimeUnit.MINUTES);
  69             final NetconfDeviceSimulator netconfDeviceSimulator = new NetconfDeviceSimulator();
  70             try {
  71                 final List<Integer> openDevices = netconfDeviceSimulator.start(params);
  72                 if (openDevices.size() == 0) {
  73                     root.error("Failed to start any simulated devices, exiting...");
  74                     System.exit(1);
  75                 }
  76                 if (params.distroFolder != null) {
  77                     final Main.ConfigGenerator configGenerator = new Main.ConfigGenerator(params.distroFolder, openDevices);
  78                     final List<File> generated = configGenerator.generate(
  79                             params.ssh, params.generateConfigBatchSize,
  80                             params.generateConfigsTimeout, params.generateConfigsAddress,
  81                             params.devicesPerPort);
  82                     configGenerator.updateFeatureFile(generated);
  83                     configGenerator.changeLoadOrder();
  84                 }
  85             } catch (final Exception e) {
  86                 root.error("Unhandled exception", e);
  87                 netconfDeviceSimulator.close();
  88                 System.exit(1);
  89             }
  90
  91             root.warn(params.distroFolder.getAbsolutePath());
  92             try {
  93                 runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/start");
  94                 String status;
  95                 do {
  96                     final Process exec = runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/status");
  97                     try {
  98                         Thread.sleep(2000l);
  99                     } catch (InterruptedException e) {
 100                         root.warn("Failed to sleep", e);
 101                     }
 102                     status = CharStreams.toString(new BufferedReader(new InputStreamReader(exec.getInputStream())));
 103                     root.warn("Current status: {}", status);
 104                 } while (!status.startsWith("Running ..."));
 105                 root.warn("Doing feature install {}", params.distroFolder.getAbsolutePath() + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 106                 final Process featureInstall = runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 107                 root.warn(CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getInputStream()))));
 108                 root.warn(CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getErrorStream()))));
 109
 110             } catch (IOException e) {
 111                 root.warn("Failed to start karaf", e);
 112                 System.exit(1);
 113             }
 114
 115             root.warn("Karaf started, starting stopwatch");
 116             stopwatch.start();
 117
 118             try {
 119                 executor.schedule(new ScaleVerifyCallable(netconfDeviceSimulator, params.deviceCount), retryDelay, TimeUnit.SECONDS);
 120                 root.warn("First callable scheduled");
 121                 semaphore.acquire();
 122                 root.warn("semaphore released");
 123             } catch (InterruptedException e) {
 124                 throw new RuntimeException(e);
 125             }
 126
 127             timeoutGuardFuture.cancel(false);
 128             params.deviceCount += deviceStep;
 129             netconfDeviceSimulator.close();
 130             stopwatch.reset();
 131
 132             cleanup(runtime, params);
 133         }
 134     }
 135
 136     private static void cleanup(final Runtime runtime, final Params params) {
 137         try {
 138             stopKaraf(runtime, params);
 139             deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 140
 141         } catch (IOException | InterruptedException e) {
 142             root.warn("Failed to stop karaf", e);
 143             System.exit(1);
 144         }
 145     }
 146
 147     private static void stopKaraf(final Runtime runtime, final Params params) throws IOException, InterruptedException {
 148         root.info("Stopping karaf and sleeping for 10 sec..");
 149         String controllerPid = "";
 150         do {
 151
 152             final Process pgrep = runtime.exec("pgrep -f org.apache.karaf.main.Main");
 153
 154             controllerPid = CharStreams.toString(new BufferedReader(new InputStreamReader(pgrep.getInputStream())));
 155             root.warn(controllerPid);
 156             runtime.exec("kill -9 " + controllerPid);
 157
 158             Thread.sleep(10000l);
 159         } while (!controllerPid.isEmpty());
 160         deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 161     }
 162
 163     private static void deleteFolder(File folder) {
 164         File[] files = folder.listFiles();
 165         if(files!=null) { //some JVMs return null for empty dirs
 166             for(File f: files) {
 167                 if(f.isDirectory()) {
 168                     deleteFolder(f);
 169                 } else {
 170                     f.delete();
 171                 }
 172             }
 173         }
 174         folder.delete();
 175     }
 176
 177     private static Params parseArgs(final String[] args, final ArgumentParser parser) {
 178         final Params parameters = new Params();
 179         try {
 180             parser.parseArgs(args, parameters);
 181             return parameters;
 182         } catch (ArgumentParserException e) {
 183             parser.handleError(e);
 184         }
 185
 186         System.exit(1);
 187         return null;
 188     }
 189
 190     private static class ScaleVerifyCallable implements Callable {
 191
 192         private static final Logger LOG = LoggerFactory.getLogger(ScaleVerifyCallable.class);
 193
 194         private static final String RESTCONF_URL = "http://127.0.0.1:8181/restconf/operational/network-topology:network-topology/topology/topology-netconf/";
 195         private static final Pattern PATTERN = Pattern.compile("connected");
 196
 197         private final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new Builder()
 198                 .setConnectTimeout(Integer.MAX_VALUE)
 199                 .setRequestTimeout(Integer.MAX_VALUE)
 200                 .setAllowPoolingConnections(true)
 201                 .build());
 202         private final NetconfDeviceSimulator simulator;
 203         private final int deviceCount;
 204         private final Request request;
 205
 206         public ScaleVerifyCallable(final NetconfDeviceSimulator simulator, final int deviceCount) {
 207             LOG.info("New callable created");
 208             this.simulator = simulator;
 209             this.deviceCount = deviceCount;
 210             AsyncHttpClient.BoundRequestBuilder requestBuilder = asyncHttpClient.prepareGet(RESTCONF_URL)
 211                     .addHeader("content-type", "application/xml")
 212                     .addHeader("Accept", "application/xml")
 213                     .setRequestTimeout(Integer.MAX_VALUE);
 214             request = requestBuilder.build();
 215         }
 216
 217         @Override
 218         public Object call() throws Exception {
 219             try {
 220                 final Response response = asyncHttpClient.executeRequest(request).get();
 221
 222                 if (response.getStatusCode() != 200 && response.getStatusCode() != 204) {
 223                     LOG.warn("Request failed, status code: {}", response.getStatusCode() + response.getStatusText());
 224                     executor.schedule(new ScaleVerifyCallable(simulator, deviceCount), retryDelay, TimeUnit.SECONDS);
 225                 } else {
 226                     final String body = response.getResponseBody();
 227                     final Matcher matcher = PATTERN.matcher(body);
 228                     int count = 0;
 229                     while (matcher.find()) {
 230                         count++;
 231                     }
 232                     RESULTS_LOG.info("Currently connected devices : {} out of {}, time elapsed: {}", count, deviceCount + 1, stopwatch);
 233                     if (count != deviceCount + 1) {
 234                         executor.schedule(new ScaleVerifyCallable(simulator, deviceCount), retryDelay, TimeUnit.SECONDS);
 235                     } else {
 236                         stopwatch.stop();
 237                         RESULTS_LOG.info("All devices connected in {}", stopwatch);
 238                         semaphore.release();
 239                     }
 240                 }
 241             } catch (ConnectException | ExecutionException e) {
 242                 LOG.warn("Failed to connect to Restconf, is the controller running?", e);
 243                 executor.schedule(new ScaleVerifyCallable(simulator, deviceCount), retryDelay, TimeUnit.SECONDS);
 244             }
 245             return null;
 246         }
 247     }
 248
 249     private static class TimeoutGuard implements Callable {
 250
 251         @Override
 252         public Object call() throws Exception {
 253             RESULTS_LOG.warn("Timeout for scale test reached after: {} ..aborting", stopwatch);
 254             root.warn("Timeout for scale test reached after: {} ..aborting", stopwatch);
 255             System.exit(0);
 256             return null;
 257         }
 258     }
 259
 260     public static class LoggingWrapperExecutor extends ScheduledThreadPoolExecutor {
 261
 262         public LoggingWrapperExecutor(int corePoolSize) {
 263             super(corePoolSize);
 264         }
 265
 266         @Override
 267         public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
 268             return super.schedule(wrapCallable(callable), delay, unit);
 269         }
 270
 271         private Callable wrapCallable(Callable callable) {
 272             return new LogOnExceptionCallable(callable);
 273         }
 274
 275         private class LogOnExceptionCallable implements Callable {
 276             private Callable theCallable;
 277
 278             public LogOnExceptionCallable(Callable theCallable) {
 279                 super();
 280                 this.theCallable = theCallable;
 281             }
 282
 283             @Override
 284             public Object call() throws Exception {
 285                 try {
 286                     theCallable.call();
 287                     return null;
 288                 } catch (Exception e) {
 289                     // log
 290                     root.warn("error in executing: " + theCallable + ". It will no longer be run!", e);
 291
 292                     // rethrow so that the executor can do it's thing
 293                     throw new RuntimeException(e);
 294                 }
 295             }
 296         }
 297     }
 298 }