netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/ScaleUtil.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.test.tool;
  10
  11 import ch.qos.logback.classic.Level;
  12 import com.google.common.base.Stopwatch;
  13 import com.google.common.io.CharStreams;
  14 import com.ning.http.client.AsyncHttpClient;
  15 import com.ning.http.client.AsyncHttpClientConfig.Builder;
  16 import com.ning.http.client.Request;
  17 import com.ning.http.client.Response;
  18 import java.io.BufferedReader;
  19 import java.io.File;
  20 import java.io.IOException;
  21 import java.io.InputStreamReader;
  22 import java.net.ConnectException;
  23 import java.util.List;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.ScheduledExecutorService;
  27 import java.util.concurrent.ScheduledFuture;
  28 import java.util.concurrent.ScheduledThreadPoolExecutor;
  29 import java.util.concurrent.Semaphore;
  30 import java.util.concurrent.TimeUnit;
  31 import java.util.regex.Matcher;
  32 import java.util.regex.Pattern;
  33 import net.sourceforge.argparse4j.inf.ArgumentParser;
  34 import net.sourceforge.argparse4j.inf.ArgumentParserException;
  35 import org.opendaylight.netconf.test.tool.config.Configuration;
  36 import org.opendaylight.netconf.test.tool.config.ConfigurationBuilder;
  37 import org.slf4j.Logger;
  38 import org.slf4j.LoggerFactory;
  39
  40 public class ScaleUtil {
  41     private static final ScheduledExecutorService EXECUTOR = new LoggingWrapperExecutor(4);
  42     private static final Semaphore SEMAPHORE = new Semaphore(0);
  43     private static final Stopwatch STOPWATCH = Stopwatch.createUnstarted();
  44
  45     private static final long TIMEOUT = 20L;
  46     private static final long RETRY_DELAY = 10L;
  47     private static final int DEVICE_STEP = 1000;
  48
  49     private static ch.qos.logback.classic.Logger root;
  50     private static Logger resultsLog;
  51
  52     @SuppressWarnings("checkstyle:illegalCatch")
  53     public static void main(final String[] args) {
  54         final TesttoolParameters params = TesttoolParameters.parseArgs(args, TesttoolParameters.getParser());
  55
  56         setUpLoggers(params);
  57
  58         // cleanup at the start in case controller was already running
  59         final Runtime runtime = Runtime.getRuntime();
  60         cleanup(runtime, params);
  61
  62         while (true) {
  63             root.warn("Starting scale test with {} devices", params.deviceCount);
  64             final ScheduledFuture timeoutGuardFuture = EXECUTOR.schedule(new TimeoutGuard(), TIMEOUT, TimeUnit.MINUTES);
  65             final Configuration configuration = new ConfigurationBuilder().from(params).build();
  66             final NetconfDeviceSimulator netconfDeviceSimulator = new NetconfDeviceSimulator(configuration);
  67             try {
  68                 final List<Integer> openDevices = netconfDeviceSimulator.start();
  69                 if (openDevices.size() == 0) {
  70                     root.error("Failed to start any simulated devices, exiting...");
  71                     System.exit(1);
  72                 }
  73                 if (params.distroFolder != null) {
  74                     final Main.ConfigGenerator configGenerator = new Main.ConfigGenerator(
  75                         params.distroFolder, openDevices);
  76                     final List<File> generated = configGenerator.generate(
  77                             params.ssh, params.generateConfigBatchSize,
  78                             params.generateConfigsTimeout, params.generateConfigsAddress,
  79                             params.devicesPerPort);
  80                     configGenerator.updateFeatureFile(generated);
  81                     configGenerator.changeLoadOrder();
  82                 }
  83             } catch (final Exception e) {
  84                 root.error("Unhandled exception", e);
  85                 netconfDeviceSimulator.close();
  86                 System.exit(1);
  87             }
  88
  89             root.warn(params.distroFolder.getAbsolutePath());
  90             try {
  91                 runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/start");
  92                 String status;
  93                 do {
  94                     final Process exec = runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/status");
  95                     try {
  96                         Thread.sleep(2000L);
  97                     } catch (InterruptedException e) {
  98                         root.warn("Failed to sleep", e);
  99                     }
 100                     status = CharStreams.toString(new BufferedReader(new InputStreamReader(exec.getInputStream())));
 101                     root.warn("Current status: {}", status);
 102                 } while (!status.startsWith("Running ..."));
 103                 root.warn("Doing feature install {}", params.distroFolder.getAbsolutePath()
 104                     + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 105                 final Process featureInstall = runtime.exec(params.distroFolder.getAbsolutePath()
 106                     + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 107                 root.warn(
 108                     CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getInputStream()))));
 109                 root.warn(
 110                     CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getErrorStream()))));
 111
 112             } catch (IOException e) {
 113                 root.warn("Failed to start karaf", e);
 114                 System.exit(1);
 115             }
 116
 117             root.warn("Karaf started, starting stopwatch");
 118             STOPWATCH.start();
 119
 120             try {
 121                 EXECUTOR.schedule(
 122                     new ScaleVerifyCallable(netconfDeviceSimulator, params.deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 123                 root.warn("First callable scheduled");
 124                 SEMAPHORE.acquire();
 125                 root.warn("semaphore released");
 126             } catch (InterruptedException e) {
 127                 throw new RuntimeException(e);
 128             }
 129
 130             timeoutGuardFuture.cancel(false);
 131             params.deviceCount += DEVICE_STEP;
 132             netconfDeviceSimulator.close();
 133             STOPWATCH.reset();
 134
 135             cleanup(runtime, params);
 136         }
 137     }
 138
 139     private static void setUpLoggers(final TesttoolParameters params) {
 140         System.setProperty("log_file_name", "scale-util.log");
 141
 142         root = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
 143         root.setLevel(params.debug ? Level.DEBUG : Level.INFO);
 144         resultsLog = LoggerFactory.getLogger("results");
 145     }
 146
 147     private static void cleanup(final Runtime runtime, final TesttoolParameters params) {
 148         try {
 149             stopKaraf(runtime, params);
 150             deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 151
 152         } catch (IOException | InterruptedException e) {
 153             root.warn("Failed to stop karaf", e);
 154             System.exit(1);
 155         }
 156     }
 157
 158     private static void stopKaraf(final Runtime runtime, final TesttoolParameters params)
 159             throws IOException, InterruptedException {
 160         root.info("Stopping karaf and sleeping for 10 sec..");
 161         String controllerPid = "";
 162         do {
 163             final Process pgrep = runtime.exec("pgrep -f org.apache.karaf.main.Main");
 164
 165             controllerPid = CharStreams.toString(new BufferedReader(new InputStreamReader(pgrep.getInputStream())));
 166             root.warn(controllerPid);
 167             runtime.exec("kill -9 " + controllerPid);
 168
 169             Thread.sleep(10000L);
 170         } while (!controllerPid.isEmpty());
 171         deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 172     }
 173
 174     private static void deleteFolder(File folder) {
 175         File[] files = folder.listFiles();
 176         if (files != null) { //some JVMs return null for empty dirs
 177             for (File f : files) {
 178                 if (f.isDirectory()) {
 179                     deleteFolder(f);
 180                 } else {
 181                     f.delete();
 182                 }
 183             }
 184         }
 185         folder.delete();
 186     }
 187
 188     private static TesttoolParameters parseArgs(final String[] args, final ArgumentParser parser) {
 189         final TesttoolParameters parameters = new TesttoolParameters();
 190         try {
 191             parser.parseArgs(args, parameters);
 192             return parameters;
 193         } catch (ArgumentParserException e) {
 194             parser.handleError(e);
 195         }
 196
 197         System.exit(1);
 198         return null;
 199     }
 200
 201     private static class ScaleVerifyCallable implements Callable {
 202         private static final Logger LOG = LoggerFactory.getLogger(ScaleVerifyCallable.class);
 203
 204         private static final String RESTCONF_URL
 205                 = "http://127.0.0.1:8181/restconf/operational/network-topology:network-topology/topology/topology-netconf/";
 206         private static final Pattern PATTERN = Pattern.compile("connected");
 207
 208         private final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new Builder()
 209                 .setConnectTimeout(Integer.MAX_VALUE)
 210                 .setRequestTimeout(Integer.MAX_VALUE)
 211                 .setAllowPoolingConnections(true)
 212                 .build());
 213         private final NetconfDeviceSimulator simulator;
 214         private final int deviceCount;
 215         private final Request request;
 216
 217         ScaleVerifyCallable(final NetconfDeviceSimulator simulator, final int deviceCount) {
 218             LOG.info("New callable created");
 219             this.simulator = simulator;
 220             this.deviceCount = deviceCount;
 221             AsyncHttpClient.BoundRequestBuilder requestBuilder = asyncHttpClient.prepareGet(RESTCONF_URL)
 222                     .addHeader("content-type", "application/xml")
 223                     .addHeader("Accept", "application/xml")
 224                     .setRequestTimeout(Integer.MAX_VALUE);
 225             request = requestBuilder.build();
 226         }
 227
 228         @Override
 229         public Object call() throws Exception {
 230             try {
 231                 final Response response = asyncHttpClient.executeRequest(request).get();
 232
 233                 if (response.getStatusCode() != 200 && response.getStatusCode() != 204) {
 234                     LOG.warn("Request failed, status code: {}", response.getStatusCode() + response.getStatusText());
 235                     EXECUTOR.schedule(new ScaleVerifyCallable(simulator, deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 236                 } else {
 237                     final String body = response.getResponseBody();
 238                     final Matcher matcher = PATTERN.matcher(body);
 239                     int count = 0;
 240                     while (matcher.find()) {
 241                         count++;
 242                     }
 243                     resultsLog.info("Currently connected devices : {} out of {}, time elapsed: {}",
 244                         count, deviceCount + 1, STOPWATCH);
 245                     if (count != deviceCount + 1) {
 246                         EXECUTOR.schedule(
 247                             new ScaleVerifyCallable(simulator, deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 248                     } else {
 249                         STOPWATCH.stop();
 250                         resultsLog.info("All devices connected in {}", STOPWATCH);
 251                         SEMAPHORE.release();
 252                     }
 253                 }
 254             } catch (ConnectException | ExecutionException e) {
 255                 LOG.warn("Failed to connect to Restconf, is the controller running?", e);
 256                 EXECUTOR.schedule(new ScaleVerifyCallable(simulator, deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 257             }
 258             return null;
 259         }
 260     }
 261
 262     private static class TimeoutGuard implements Callable {
 263         @Override
 264         public Object call() throws Exception {
 265             resultsLog.warn("Timeout for scale test reached after: {} ..aborting", STOPWATCH);
 266             root.warn("Timeout for scale test reached after: {} ..aborting", STOPWATCH);
 267             System.exit(0);
 268             return null;
 269         }
 270     }
 271
 272     @SuppressWarnings("checkstyle:illegalCatch")
 273     public static class LoggingWrapperExecutor extends ScheduledThreadPoolExecutor {
 274         public LoggingWrapperExecutor(int corePoolSize) {
 275             super(corePoolSize);
 276         }
 277
 278         @Override
 279         public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
 280             return super.schedule(wrapCallable(callable), delay, unit);
 281         }
 282
 283         private Callable wrapCallable(Callable callable) {
 284             return new LogOnExceptionCallable(callable);
 285         }
 286
 287         private class LogOnExceptionCallable implements Callable {
 288             private Callable theCallable;
 289
 290             LogOnExceptionCallable(Callable theCallable) {
 291                 super();
 292                 this.theCallable = theCallable;
 293             }
 294
 295             @Override
 296             public Object call() throws Exception {
 297                 try {
 298                     theCallable.call();
 299                     return null;
 300                 } catch (Exception e) {
 301                     // log
 302                     root.warn("error in executing: " + theCallable + ". It will no longer be run!", e);
 303
 304                     // rethrow so that the executor can do it's thing
 305                     throw new RuntimeException(e);
 306                 }
 307             }
 308         }
 309     }
 310 }