netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/ScaleUtil.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.test.tool;
  10
  11 import ch.qos.logback.classic.Level;
  12 import ch.qos.logback.classic.LoggerContext;
  13 import ch.qos.logback.classic.util.ContextInitializer;
  14 import ch.qos.logback.core.joran.spi.JoranException;
  15 import ch.qos.logback.core.util.StatusPrinter;
  16 import com.google.common.base.Stopwatch;
  17 import com.google.common.io.CharStreams;
  18 import com.ning.http.client.AsyncHttpClient;
  19 import com.ning.http.client.AsyncHttpClientConfig.Builder;
  20 import com.ning.http.client.Request;
  21 import com.ning.http.client.Response;
  22 import java.io.BufferedReader;
  23 import java.io.File;
  24 import java.io.IOException;
  25 import java.io.InputStreamReader;
  26 import java.net.ConnectException;
  27 import java.util.List;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.ExecutionException;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.ScheduledFuture;
  32 import java.util.concurrent.ScheduledThreadPoolExecutor;
  33 import java.util.concurrent.Semaphore;
  34 import java.util.concurrent.TimeUnit;
  35 import java.util.regex.Matcher;
  36 import java.util.regex.Pattern;
  37 import net.sourceforge.argparse4j.inf.ArgumentParser;
  38 import net.sourceforge.argparse4j.inf.ArgumentParserException;
  39 import org.slf4j.Logger;
  40 import org.slf4j.LoggerFactory;
  41
  42 public class ScaleUtil {
  43     private static Logger RESULTS_LOG ;
  44     private static final ScheduledExecutorService executor = new LoggingWrapperExecutor(4);
  45
  46     private static final int deviceStep = 1000;
  47     private static final long retryDelay = 10l;
  48     private static final long timeout = 20l;
  49
  50     private static final Stopwatch stopwatch = Stopwatch.createUnstarted();
  51
  52     private static ScheduledFuture timeoutGuardFuture;
  53     private static ch.qos.logback.classic.Logger root;
  54     private static final Semaphore semaphore = new Semaphore(0);
  55
  56     public static void main(final String[] args) {
  57         final TesttoolParameters params = TesttoolParameters.parseArgs(args, TesttoolParameters.getParser());
  58
  59         setUpLoggers(params);
  60
  61         // cleanup at the start in case controller was already running
  62         final Runtime runtime = Runtime.getRuntime();
  63         cleanup(runtime, params);
  64
  65         while (true) {
  66             root.warn("Starting scale test with {} devices", params.deviceCount);
  67             timeoutGuardFuture = executor.schedule(new TimeoutGuard(), timeout, TimeUnit.MINUTES);
  68             final NetconfDeviceSimulator netconfDeviceSimulator = new NetconfDeviceSimulator();
  69             try {
  70                 final List<Integer> openDevices = netconfDeviceSimulator.start(params);
  71                 if (openDevices.size() == 0) {
  72                     root.error("Failed to start any simulated devices, exiting...");
  73                     System.exit(1);
  74                 }
  75                 if (params.distroFolder != null) {
  76                     final Main.ConfigGenerator configGenerator = new Main.ConfigGenerator(params.distroFolder, openDevices);
  77                     final List<File> generated = configGenerator.generate(
  78                             params.ssh, params.generateConfigBatchSize,
  79                             params.generateConfigsTimeout, params.generateConfigsAddress,
  80                             params.devicesPerPort);
  81                     configGenerator.updateFeatureFile(generated);
  82                     configGenerator.changeLoadOrder();
  83                 }
  84             } catch (final Exception e) {
  85                 root.error("Unhandled exception", e);
  86                 netconfDeviceSimulator.close();
  87                 System.exit(1);
  88             }
  89
  90             root.warn(params.distroFolder.getAbsolutePath());
  91             try {
  92                 runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/start");
  93                 String status;
  94                 do {
  95                     final Process exec = runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/status");
  96                     try {
  97                         Thread.sleep(2000l);
  98                     } catch (InterruptedException e) {
  99                         root.warn("Failed to sleep", e);
 100                     }
 101                     status = CharStreams.toString(new BufferedReader(new InputStreamReader(exec.getInputStream())));
 102                     root.warn("Current status: {}", status);
 103                 } while (!status.startsWith("Running ..."));
 104                 root.warn("Doing feature install {}", params.distroFolder.getAbsolutePath() + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 105                 final Process featureInstall = runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 106                 root.warn(CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getInputStream()))));
 107                 root.warn(CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getErrorStream()))));
 108
 109             } catch (IOException e) {
 110                 root.warn("Failed to start karaf", e);
 111                 System.exit(1);
 112             }
 113
 114             root.warn("Karaf started, starting stopwatch");
 115             stopwatch.start();
 116
 117             try {
 118                 executor.schedule(new ScaleVerifyCallable(netconfDeviceSimulator, params.deviceCount), retryDelay, TimeUnit.SECONDS);
 119                 root.warn("First callable scheduled");
 120                 semaphore.acquire();
 121                 root.warn("semaphore released");
 122             } catch (InterruptedException e) {
 123                 throw new RuntimeException(e);
 124             }
 125
 126             timeoutGuardFuture.cancel(false);
 127             params.deviceCount += deviceStep;
 128             netconfDeviceSimulator.close();
 129             stopwatch.reset();
 130
 131             cleanup(runtime, params);
 132         }
 133     }
 134
 135     private static void setUpLoggers(final TesttoolParameters params) {
 136         System.setProperty("log_file_name", "scale-util.log");
 137
 138         root = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
 139         root.setLevel(params.debug ? Level.DEBUG : Level.INFO);
 140         RESULTS_LOG = LoggerFactory.getLogger("results");
 141     }
 142
 143     private static void cleanup(final Runtime runtime, final TesttoolParameters params) {
 144         try {
 145             stopKaraf(runtime, params);
 146             deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 147
 148         } catch (IOException | InterruptedException e) {
 149             root.warn("Failed to stop karaf", e);
 150             System.exit(1);
 151         }
 152     }
 153
 154     private static void stopKaraf(final Runtime runtime, final TesttoolParameters params) throws IOException, InterruptedException {
 155         root.info("Stopping karaf and sleeping for 10 sec..");
 156         String controllerPid = "";
 157         do {
 158
 159             final Process pgrep = runtime.exec("pgrep -f org.apache.karaf.main.Main");
 160
 161             controllerPid = CharStreams.toString(new BufferedReader(new InputStreamReader(pgrep.getInputStream())));
 162             root.warn(controllerPid);
 163             runtime.exec("kill -9 " + controllerPid);
 164
 165             Thread.sleep(10000l);
 166         } while (!controllerPid.isEmpty());
 167         deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 168     }
 169
 170     private static void deleteFolder(File folder) {
 171         File[] files = folder.listFiles();
 172         if(files!=null) { //some JVMs return null for empty dirs
 173             for(File f: files) {
 174                 if(f.isDirectory()) {
 175                     deleteFolder(f);
 176                 } else {
 177                     f.delete();
 178                 }
 179             }
 180         }
 181         folder.delete();
 182     }
 183
 184     private static TesttoolParameters parseArgs(final String[] args, final ArgumentParser parser) {
 185         final TesttoolParameters parameters = new TesttoolParameters();
 186         try {
 187             parser.parseArgs(args, parameters);
 188             return parameters;
 189         } catch (ArgumentParserException e) {
 190             parser.handleError(e);
 191         }
 192
 193         System.exit(1);
 194         return null;
 195     }
 196
 197     private static class ScaleVerifyCallable implements Callable {
 198
 199         private static final Logger LOG = LoggerFactory.getLogger(ScaleVerifyCallable.class);
 200
 201         private static final String RESTCONF_URL = "http://127.0.0.1:8181/restconf/operational/network-topology:network-topology/topology/topology-netconf/";
 202         private static final Pattern PATTERN = Pattern.compile("connected");
 203
 204         private final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new Builder()
 205                 .setConnectTimeout(Integer.MAX_VALUE)
 206                 .setRequestTimeout(Integer.MAX_VALUE)
 207                 .setAllowPoolingConnections(true)
 208                 .build());
 209         private final NetconfDeviceSimulator simulator;
 210         private final int deviceCount;
 211         private final Request request;
 212
 213         public ScaleVerifyCallable(final NetconfDeviceSimulator simulator, final int deviceCount) {
 214             LOG.info("New callable created");
 215             this.simulator = simulator;
 216             this.deviceCount = deviceCount;
 217             AsyncHttpClient.BoundRequestBuilder requestBuilder = asyncHttpClient.prepareGet(RESTCONF_URL)
 218                     .addHeader("content-type", "application/xml")
 219                     .addHeader("Accept", "application/xml")
 220                     .setRequestTimeout(Integer.MAX_VALUE);
 221             request = requestBuilder.build();
 222         }
 223
 224         @Override
 225         public Object call() throws Exception {
 226             try {
 227                 final Response response = asyncHttpClient.executeRequest(request).get();
 228
 229                 if (response.getStatusCode() != 200 && response.getStatusCode() != 204) {
 230                     LOG.warn("Request failed, status code: {}", response.getStatusCode() + response.getStatusText());
 231                     executor.schedule(new ScaleVerifyCallable(simulator, deviceCount), retryDelay, TimeUnit.SECONDS);
 232                 } else {
 233                     final String body = response.getResponseBody();
 234                     final Matcher matcher = PATTERN.matcher(body);
 235                     int count = 0;
 236                     while (matcher.find()) {
 237                         count++;
 238                     }
 239                     RESULTS_LOG.info("Currently connected devices : {} out of {}, time elapsed: {}", count, deviceCount + 1, stopwatch);
 240                     if (count != deviceCount + 1) {
 241                         executor.schedule(new ScaleVerifyCallable(simulator, deviceCount), retryDelay, TimeUnit.SECONDS);
 242                     } else {
 243                         stopwatch.stop();
 244                         RESULTS_LOG.info("All devices connected in {}", stopwatch);
 245                         semaphore.release();
 246                     }
 247                 }
 248             } catch (ConnectException | ExecutionException e) {
 249                 LOG.warn("Failed to connect to Restconf, is the controller running?", e);
 250                 executor.schedule(new ScaleVerifyCallable(simulator, deviceCount), retryDelay, TimeUnit.SECONDS);
 251             }
 252             return null;
 253         }
 254     }
 255
 256     private static class TimeoutGuard implements Callable {
 257
 258         @Override
 259         public Object call() throws Exception {
 260             RESULTS_LOG.warn("Timeout for scale test reached after: {} ..aborting", stopwatch);
 261             root.warn("Timeout for scale test reached after: {} ..aborting", stopwatch);
 262             System.exit(0);
 263             return null;
 264         }
 265     }
 266
 267     public static class LoggingWrapperExecutor extends ScheduledThreadPoolExecutor {
 268
 269         public LoggingWrapperExecutor(int corePoolSize) {
 270             super(corePoolSize);
 271         }
 272
 273         @Override
 274         public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
 275             return super.schedule(wrapCallable(callable), delay, unit);
 276         }
 277
 278         private Callable wrapCallable(Callable callable) {
 279             return new LogOnExceptionCallable(callable);
 280         }
 281
 282         private class LogOnExceptionCallable implements Callable {
 283             private Callable theCallable;
 284
 285             public LogOnExceptionCallable(Callable theCallable) {
 286                 super();
 287                 this.theCallable = theCallable;
 288             }
 289
 290             @Override
 291             public Object call() throws Exception {
 292                 try {
 293                     theCallable.call();
 294                     return null;
 295                 } catch (Exception e) {
 296                     // log
 297                     root.warn("error in executing: " + theCallable + ". It will no longer be run!", e);
 298
 299                     // rethrow so that the executor can do it's thing
 300                     throw new RuntimeException(e);
 301                 }
 302             }
 303         }
 304     }
 305 }