netconf/tools/netconf-testtool/src/main/java/org/opendaylight/netconf/test/tool/ScaleUtil.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netconf.test.tool;
  10
  11 import ch.qos.logback.classic.Level;
  12 import com.google.common.base.Stopwatch;
  13 import com.google.common.io.CharStreams;
  14 import com.ning.http.client.AsyncHttpClient;
  15 import com.ning.http.client.AsyncHttpClientConfig.Builder;
  16 import com.ning.http.client.Request;
  17 import com.ning.http.client.Response;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.io.BufferedReader;
  20 import java.io.File;
  21 import java.io.IOException;
  22 import java.io.InputStreamReader;
  23 import java.net.ConnectException;
  24 import java.util.List;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.ScheduledExecutorService;
  28 import java.util.concurrent.ScheduledFuture;
  29 import java.util.concurrent.ScheduledThreadPoolExecutor;
  30 import java.util.concurrent.Semaphore;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.regex.Matcher;
  33 import java.util.regex.Pattern;
  34 import org.opendaylight.netconf.test.tool.config.Configuration;
  35 import org.opendaylight.netconf.test.tool.config.ConfigurationBuilder;
  36 import org.slf4j.Logger;
  37 import org.slf4j.LoggerFactory;
  38
  39 @SuppressFBWarnings({"DM_EXIT", "DM_DEFAULT_ENCODING", "SLF4J_LOGGER_SHOULD_BE_FINAL"})
  40 public final class ScaleUtil {
  41     private static final ScheduledExecutorService EXECUTOR = new LoggingWrapperExecutor(4);
  42     private static final Semaphore SEMAPHORE = new Semaphore(0);
  43     private static final Stopwatch STOPWATCH = Stopwatch.createUnstarted();
  44
  45     private static final long TIMEOUT = 20L;
  46     private static final long RETRY_DELAY = 10L;
  47     private static final int DEVICE_STEP = 1000;
  48
  49     private static ch.qos.logback.classic.Logger root;
  50     private static Logger resultsLog;
  51
  52     private ScaleUtil() {
  53     }
  54
  55     @SuppressWarnings("checkstyle:illegalCatch")
  56     public static void main(final String[] args) {
  57         final TesttoolParameters params = TesttoolParameters.parseArgs(args, TesttoolParameters.getParser());
  58
  59         setUpLoggers(params);
  60
  61         // cleanup at the start in case controller was already running
  62         final Runtime runtime = Runtime.getRuntime();
  63         cleanup(runtime, params);
  64
  65         while (true) {
  66             root.warn("Starting scale test with {} devices", params.deviceCount);
  67             final ScheduledFuture<?> timeoutGuardFuture = EXECUTOR.schedule(new TimeoutGuard(), TIMEOUT,
  68                 TimeUnit.MINUTES);
  69             final Configuration configuration = new ConfigurationBuilder().from(params).build();
  70             final NetconfDeviceSimulator netconfDeviceSimulator = new NetconfDeviceSimulator(configuration);
  71             try {
  72                 final List<Integer> openDevices = netconfDeviceSimulator.start();
  73                 if (openDevices.size() == 0) {
  74                     root.error("Failed to start any simulated devices, exiting...");
  75                     System.exit(1);
  76                 }
  77
  78                 if (params.distroFolder == null) {
  79                     root.error("Distro folder is not set, exiting...");
  80                     System.exit(1);
  81                 }
  82
  83                 final Main.ConfigGenerator configGenerator = new Main.ConfigGenerator(
  84                         params.distroFolder, openDevices);
  85                 final List<File> generated = configGenerator.generate(
  86                         params.ssh, params.generateConfigBatchSize,
  87                         params.generateConfigsTimeout, params.generateConfigsAddress,
  88                         params.devicesPerPort);
  89                 configGenerator.updateFeatureFile(generated);
  90                 configGenerator.changeLoadOrder();
  91             } catch (final Exception e) {
  92                 root.error("Unhandled exception", e);
  93                 netconfDeviceSimulator.close();
  94                 System.exit(1);
  95             }
  96
  97             root.warn(params.distroFolder.getAbsolutePath());
  98             try {
  99                 runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/start");
 100                 String status;
 101                 do {
 102                     final Process exec = runtime.exec(params.distroFolder.getAbsolutePath() + "/bin/status");
 103                     try {
 104                         Thread.sleep(2000L);
 105                     } catch (InterruptedException e) {
 106                         root.warn("Failed to sleep", e);
 107                     }
 108                     status = CharStreams.toString(new BufferedReader(new InputStreamReader(exec.getInputStream())));
 109                     root.warn("Current status: {}", status);
 110                 } while (!status.startsWith("Running ..."));
 111                 root.warn("Doing feature install {}", params.distroFolder.getAbsolutePath()
 112                     + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 113                 final Process featureInstall = runtime.exec(params.distroFolder.getAbsolutePath()
 114                     + "/bin/client -u karaf feature:install odl-restconf-noauth odl-netconf-connector-all");
 115                 root.warn(
 116                     CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getInputStream()))));
 117                 root.warn(
 118                     CharStreams.toString(new BufferedReader(new InputStreamReader(featureInstall.getErrorStream()))));
 119
 120             } catch (IOException e) {
 121                 root.warn("Failed to start karaf", e);
 122                 System.exit(1);
 123             }
 124
 125             root.warn("Karaf started, starting stopwatch");
 126             STOPWATCH.start();
 127
 128             try {
 129                 EXECUTOR.schedule(
 130                     new ScaleVerifyCallable(netconfDeviceSimulator, params.deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 131                 root.warn("First callable scheduled");
 132                 SEMAPHORE.acquire();
 133                 root.warn("semaphore released");
 134             } catch (InterruptedException e) {
 135                 throw new RuntimeException(e);
 136             }
 137
 138             timeoutGuardFuture.cancel(false);
 139             params.deviceCount += DEVICE_STEP;
 140             netconfDeviceSimulator.close();
 141             STOPWATCH.reset();
 142
 143             cleanup(runtime, params);
 144         }
 145     }
 146
 147     private static void setUpLoggers(final TesttoolParameters params) {
 148         System.setProperty("log_file_name", "scale-util.log");
 149
 150         root = (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
 151         root.setLevel(params.debug ? Level.DEBUG : Level.INFO);
 152         resultsLog = LoggerFactory.getLogger("results");
 153     }
 154
 155     private static void cleanup(final Runtime runtime, final TesttoolParameters params) {
 156         try {
 157             stopKaraf(runtime, params);
 158             deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 159
 160         } catch (IOException | InterruptedException e) {
 161             root.warn("Failed to stop karaf", e);
 162             System.exit(1);
 163         }
 164     }
 165
 166     private static void stopKaraf(final Runtime runtime, final TesttoolParameters params)
 167             throws IOException, InterruptedException {
 168         root.info("Stopping karaf and sleeping for 10 sec..");
 169         String controllerPid = "";
 170         do {
 171             final Process pgrep = runtime.exec("pgrep -f org.apache.karaf.main.Main");
 172
 173             controllerPid = CharStreams.toString(new BufferedReader(new InputStreamReader(pgrep.getInputStream())));
 174             root.warn(controllerPid);
 175             runtime.exec("kill -9 " + controllerPid);
 176
 177             Thread.sleep(10000L);
 178         } while (!controllerPid.isEmpty());
 179         deleteFolder(new File(params.distroFolder.getAbsoluteFile() + "/data"));
 180     }
 181
 182     private static void deleteFolder(final File folder) {
 183         File[] files = folder.listFiles();
 184         if (files != null) { //some JVMs return null for empty dirs
 185             for (File f : files) {
 186                 if (f.isDirectory()) {
 187                     deleteFolder(f);
 188                 } else {
 189                     if (!f.delete()) {
 190                         root.warn("Failed to delete {}", f);
 191                     }
 192                 }
 193             }
 194         }
 195         if (!folder.delete()) {
 196             root.warn("Failed to delete {}", folder);
 197         }
 198     }
 199
 200     private static class ScaleVerifyCallable implements Callable<Void> {
 201         private static final Logger LOG = LoggerFactory.getLogger(ScaleVerifyCallable.class);
 202
 203         private static final String RESTCONF_URL = "http://127.0.0.1:8181/restconf/operational/"
 204                 + "network-topology:network-topology/topology/topology-netconf/";
 205         private static final Pattern PATTERN = Pattern.compile("connected");
 206
 207         private final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new Builder()
 208                 .setConnectTimeout(Integer.MAX_VALUE)
 209                 .setRequestTimeout(Integer.MAX_VALUE)
 210                 .setAllowPoolingConnections(true)
 211                 .build());
 212         private final NetconfDeviceSimulator simulator;
 213         private final int deviceCount;
 214         private final Request request;
 215
 216         ScaleVerifyCallable(final NetconfDeviceSimulator simulator, final int deviceCount) {
 217             LOG.info("New callable created");
 218             this.simulator = simulator;
 219             this.deviceCount = deviceCount;
 220             AsyncHttpClient.BoundRequestBuilder requestBuilder = asyncHttpClient.prepareGet(RESTCONF_URL)
 221                     .addHeader("content-type", "application/xml")
 222                     .addHeader("Accept", "application/xml")
 223                     .setRequestTimeout(Integer.MAX_VALUE);
 224             request = requestBuilder.build();
 225         }
 226
 227         @Override
 228         public Void call() throws Exception {
 229             try {
 230                 final Response response = asyncHttpClient.executeRequest(request).get();
 231
 232                 if (response.getStatusCode() != 200 && response.getStatusCode() != 204) {
 233                     LOG.warn("Request failed, status code: {}", response.getStatusCode() + response.getStatusText());
 234                     EXECUTOR.schedule(new ScaleVerifyCallable(simulator, deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 235                 } else {
 236                     final String body = response.getResponseBody();
 237                     final Matcher matcher = PATTERN.matcher(body);
 238                     int count = 0;
 239                     while (matcher.find()) {
 240                         count++;
 241                     }
 242                     resultsLog.info("Currently connected devices : {} out of {}, time elapsed: {}",
 243                         count, deviceCount + 1, STOPWATCH);
 244                     if (count != deviceCount + 1) {
 245                         EXECUTOR.schedule(
 246                             new ScaleVerifyCallable(simulator, deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 247                     } else {
 248                         STOPWATCH.stop();
 249                         resultsLog.info("All devices connected in {}", STOPWATCH);
 250                         SEMAPHORE.release();
 251                     }
 252                 }
 253             } catch (ConnectException | ExecutionException e) {
 254                 LOG.warn("Failed to connect to Restconf, is the controller running?", e);
 255                 EXECUTOR.schedule(new ScaleVerifyCallable(simulator, deviceCount), RETRY_DELAY, TimeUnit.SECONDS);
 256             }
 257             return null;
 258         }
 259     }
 260
 261     private static class TimeoutGuard implements Callable<Void> {
 262         @Override
 263         public Void call() {
 264             resultsLog.warn("Timeout for scale test reached after: {} ..aborting", STOPWATCH);
 265             root.warn("Timeout for scale test reached after: {} ..aborting", STOPWATCH);
 266             System.exit(0);
 267             return null;
 268         }
 269     }
 270
 271     @SuppressWarnings("checkstyle:illegalCatch")
 272     public static class LoggingWrapperExecutor extends ScheduledThreadPoolExecutor {
 273         public LoggingWrapperExecutor(final int corePoolSize) {
 274             super(corePoolSize);
 275         }
 276
 277         @Override
 278         public <V> ScheduledFuture<V> schedule(final Callable<V> callable, final long delay, final TimeUnit unit) {
 279             return super.schedule(new LogOnExceptionCallable<>(callable), delay, unit);
 280         }
 281
 282         private static class LogOnExceptionCallable<T> implements Callable<T> {
 283             private final Callable<T> theCallable;
 284
 285             LogOnExceptionCallable(final Callable<T> theCallable) {
 286                 this.theCallable = theCallable;
 287             }
 288
 289             @Override
 290             public T call() {
 291                 try {
 292                     return theCallable.call();
 293                 } catch (Exception e) {
 294                     // log
 295                     root.warn("error in executing: " + theCallable + ". It will no longer be run!", e);
 296
 297                     // rethrow so that the executor can do it's thing
 298                     throw new RuntimeException(e);
 299                 }
 300             }
 301         }
 302     }
 303 }