metrics/impl/src/main/java/org/opendaylight/infrautils/metrics/internal/MetricProviderImpl.java

   1 /*
   2  * Copyright (c) 2017 - 2018 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.infrautils.metrics.internal;
   9
  10 import static com.codahale.metrics.Slf4jReporter.LoggingLevel.INFO;
  11 import static java.lang.management.ManagementFactory.getThreadMXBean;
  12 import static java.util.Objects.requireNonNull;
  13 import static java.util.concurrent.TimeUnit.MILLISECONDS;
  14 import static java.util.concurrent.TimeUnit.SECONDS;
  15
  16 import com.codahale.metrics.MetricRegistry;
  17 import com.codahale.metrics.Slf4jReporter;
  18 import com.codahale.metrics.jmx.JmxReporter;
  19 import com.codahale.metrics.jvm.BufferPoolMetricSet;
  20 import com.codahale.metrics.jvm.CachedThreadStatesGaugeSet;
  21 import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
  22 import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
  23 import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
  24 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
  25 import com.codahale.metrics.jvm.ThreadDeadlockDetector;
  26 import com.google.common.annotations.VisibleForTesting;
  27 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  28 import java.lang.management.ManagementFactory;
  29 import java.time.Duration;
  30 import java.util.Map;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import javax.annotation.Nullable;
  33 import javax.annotation.PreDestroy;
  34 import javax.inject.Singleton;
  35 import org.apache.aries.blueprint.annotation.service.Service;
  36 import org.opendaylight.infrautils.metrics.Counter;
  37 import org.opendaylight.infrautils.metrics.Labeled;
  38 import org.opendaylight.infrautils.metrics.Meter;
  39 import org.opendaylight.infrautils.metrics.MetricDescriptor;
  40 import org.opendaylight.infrautils.metrics.MetricProvider;
  41 import org.opendaylight.infrautils.metrics.Timer;
  42 import org.opendaylight.infrautils.utils.UncheckedCloseable;
  43 import org.opendaylight.infrautils.utils.function.CheckedCallable;
  44 import org.opendaylight.infrautils.utils.function.CheckedRunnable;
  45 import org.slf4j.Logger;
  46 import org.slf4j.LoggerFactory;
  47
  48 /**
  49  * Implementation of {@link MetricProvider} based on <a href="http://metrics.dropwizard.io">Coda Hale's Dropwizard Metrics</a>.
  50  *
  51  * @author Michael Vorburger.ch
  52  */
  53 @Singleton
  54 @Service(classes = MetricProvider.class)
  55 public class MetricProviderImpl implements MetricProvider {
  56
  57     private static final Logger LOG = LoggerFactory.getLogger(MetricProviderImpl.class);
  58
  59     private final Map<String, MeterImpl> meters = new ConcurrentHashMap<>();
  60     private final Map<String, CounterImpl> counters = new ConcurrentHashMap<>();
  61     private final Map<String, TimerImpl> timers = new ConcurrentHashMap<>();
  62     private final MetricRegistry registry;
  63     private final JmxReporter jmxReporter;
  64     private final Slf4jReporter slf4jReporter;
  65
  66     private volatile @Nullable MetricsFileReporter fileReporter;
  67     private volatile @Nullable ThreadsWatcher threadsWatcher;
  68
  69     public MetricProviderImpl() {
  70         this.registry = new MetricRegistry();
  71
  72         setUpJvmMetrics(registry);
  73
  74         jmxReporter = setUpJmxReporter(registry);
  75
  76         slf4jReporter = setUpSlf4jReporter(registry);
  77
  78         // TODO really get this to work in Karaf, through PAX Logging.. (it's currently NOK)
  79         // instrumentLog4jV2(registry);
  80     }
  81
  82     public MetricProviderImpl(Configuration configuration) {
  83         this();
  84         updateConfiguration(configuration);
  85     }
  86
  87     public final void updateConfiguration(Configuration configuration) {
  88         if (threadsWatcher != null) {
  89             threadsWatcher.close();
  90         }
  91         if (configuration.getThreadsWatcherIntervalMS() > 0 && (threadsWatcher == null
  92                 || configuration.getThreadsWatcherIntervalMS() != threadsWatcher.getInterval().toMillis()
  93                 || configuration.getMaxThreads() != threadsWatcher.getMaxThreads())
  94                 || configuration.getMaxThreadsMaxLogIntervalSecs()
  95                         != threadsWatcher.getMaxThreadsMaxLogInterval().getSeconds()
  96                 || configuration.getDeadlockedThreadsMaxLogIntervalSecs()
  97                         != threadsWatcher.getDeadlockedThreadsMaxLogInterval().getSeconds()) {
  98             threadsWatcher = new ThreadsWatcher(configuration.getMaxThreads(),
  99                     Duration.ofMillis(configuration.getThreadsWatcherIntervalMS()),
 100                     Duration.ofSeconds(configuration.getMaxThreadsMaxLogIntervalSecs()),
 101                     Duration.ofSeconds(configuration.getDeadlockedThreadsMaxLogIntervalSecs()));
 102             threadsWatcher.start();
 103         }
 104
 105         if (fileReporter != null) {
 106             fileReporter.close();
 107         }
 108         int fileReporterInterval = fileReporter != null ? (int)fileReporter.getInterval().getSeconds() : 0;
 109         if (fileReporterInterval != configuration.getFileReporterIntervalSecs()) {
 110             if (configuration.getFileReporterIntervalSecs() > 0) {
 111                 fileReporter = new MetricsFileReporter(registry,
 112                         Duration.ofSeconds(configuration.getFileReporterIntervalSecs()));
 113                 fileReporter.startReporter();
 114             }
 115         }
 116         LOG.info("Updated: {}", configuration);
 117     }
 118
 119     @PreDestroy
 120     public void close() {
 121         jmxReporter.close();
 122         if (fileReporter != null) {
 123             fileReporter.close();
 124         }
 125         slf4jReporter.close();
 126         if (threadsWatcher != null) {
 127             threadsWatcher.close();
 128         }
 129     }
 130
 131     @VisibleForTesting
 132     public MetricRegistry getRegistry() {
 133         return registry;
 134     }
 135
 136     private static void setUpJvmMetrics(MetricRegistry registry) {
 137         ThreadDeadlockDetector threadDeadlockDetector = new ThreadDeadlockDetector();
 138         FileDescriptorRatioGauge fileDescriptorRatioGauge = new FileDescriptorRatioGauge();
 139
 140         registry.registerAll(new GarbageCollectorMetricSet());
 141         registry.registerAll(new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()));
 142         registry.registerAll(new CachedThreadStatesGaugeSet(getThreadMXBean(), threadDeadlockDetector, 13, SECONDS));
 143         registry.registerAll(new MemoryUsageGaugeSet());
 144         registry.registerAll(new ClassLoadingGaugeSet());
 145         registry.gauge("odl.infrautils.FileDescriptorRatio", () -> fileDescriptorRatioGauge);
 146     }
 147
 148     private static JmxReporter setUpJmxReporter(MetricRegistry registry) {
 149         JmxReporter reporter = JmxReporter.forRegistry(registry)
 150                 .createsObjectNamesWith(new CustomObjectNameFactory()).build();
 151         reporter.start();
 152         LOG.info("JmxReporter started, ODL application's metrics are now available via JMX");
 153         return reporter;
 154     }
 155
 156     private static Slf4jReporter setUpSlf4jReporter(MetricRegistry registry) {
 157         Slf4jReporter slf4jReporter = Slf4jReporter.forRegistry(registry)
 158                 .convertDurationsTo(MILLISECONDS)
 159                 .convertRatesTo(SECONDS)
 160                 .outputTo(LOG)
 161                 .prefixedWith("JVM")
 162                 .withLoggingLevel(INFO)
 163                 .shutdownExecutorOnStop(true)
 164                 .build();
 165         // NB: We do intentionally *NOT* start() the Slf4jReporter to log all metrics regularly;
 166         // as that will spam the log, and we have our own file based reporting instead.
 167         // We do log system metrics once at boot up:
 168         LOG.info("One time system JVM metrics FYI; "
 169                 + "to watch continously, monitor via JMX or enable periodic file dump option");
 170         slf4jReporter.report();
 171         return slf4jReporter;
 172     }
 173
 174     // http://metrics.dropwizard.io/3.1.0/manual/log4j/
 175 //    private static void instrumentLog4jV2(MetricRegistry registry) {
 176 //        // TODO Confirm that Level ALL is a good idea?
 177 //        Level level = ALL;
 178 //
 179 //        InstrumentedAppender appender = new InstrumentedAppender(registry,
 180 //                null /* null Filter fine, because we don't use filters */,
 181 //                null /* null PatternLayout, because the layout isn't used in InstrumentedAppender */, false);
 182 //        appender.start();
 183 //
 184 //        LoggerContext context = (LoggerContext) LogManager.getContext(false);
 185 //        Configuration config = context.getConfiguration();
 186 //        config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME).addAppender(appender, level,
 187 //                null /* null Filter fine, because we don't use filters */);
 188 //        context.updateLoggers(config);
 189 //    }
 190
 191     private org.opendaylight.infrautils.metrics.Meter newOrExistingMeter(Object anchor, String id) {
 192         return meters.computeIfAbsent(id, newId -> {
 193             LOG.debug("New Meter metric: {}", id);
 194             return new MeterImpl(newId);
 195         });
 196     }
 197
 198     @Override
 199     public org.opendaylight.infrautils.metrics.Meter newMeter(Object anchor, String id) {
 200         requireNonNull(anchor, "anchor == null");
 201         checkForExistingID(id);
 202         return newOrExistingMeter(anchor, id);
 203     }
 204
 205     @Override
 206     public Meter newMeter(MetricDescriptor descriptor) {
 207         return newMeter(descriptor.anchor(), makeCodahaleID(descriptor));
 208     }
 209
 210     @Override
 211     public Labeled<Meter> newMeter(MetricDescriptor descriptor, String labelName) {
 212         return labelValue -> newOrExistingMeter(descriptor.anchor(),
 213                 makeCodahaleID(descriptor) + "{" + labelName + "=" + labelValue + "}");
 214     }
 215
 216     @Override
 217     public Labeled<Labeled<Meter>> newMeter(MetricDescriptor descriptor,
 218             String firstLabelName, String secondLabelName) {
 219         return firstLabelValue -> secondLabelValue -> newOrExistingMeter(descriptor.anchor(), makeCodahaleID(descriptor)
 220                 + "{" + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue + "}");
 221     }
 222
 223     @Override
 224     public Labeled<Labeled<Labeled<Meter>>> newMeter(MetricDescriptor descriptor,
 225             String firstLabelName, String secondLabelName, String thirdLabelName) {
 226         return firstLabelValue -> secondLabelValue -> thirdLabelValue ->
 227         newOrExistingMeter(descriptor.anchor(), makeCodahaleID(descriptor) + "{"
 228                 + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue  + ","
 229                 + thirdLabelName + "=" + thirdLabelValue + "}");
 230     }
 231
 232     @Override
 233     public Labeled<Labeled<Labeled<Labeled<Meter>>>> newMeter(MetricDescriptor descriptor,
 234             String firstLabelName, String secondLabelName, String thirdLabelName, String fourthLabelName) {
 235         return firstLabelValue -> secondLabelValue -> thirdLabelValue -> fourthLabelValue ->
 236                 newOrExistingMeter(descriptor.anchor(), makeCodahaleID(descriptor) + "{"
 237                         + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue  + ","
 238                         + thirdLabelName + "=" + thirdLabelValue + ","
 239                         + fourthLabelName + "=" + fourthLabelValue + "}");
 240     }
 241
 242     @Override
 243     public Labeled<Labeled<Labeled<Labeled<Labeled<Meter>>>>> newMeter(MetricDescriptor descriptor,
 244             String firstLabelName, String secondLabelName, String thirdLabelName,
 245             String fourthLabelName, String fifthLabelName) {
 246         return firstLabelValue -> secondLabelValue -> thirdLabelValue -> fourthLabelValue -> fifthLabelValue ->
 247                 newOrExistingMeter(descriptor.anchor(), makeCodahaleID(descriptor) + "{"
 248                         + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue  + ","
 249                         + thirdLabelName + "=" + thirdLabelValue + ","
 250                         + fourthLabelName + "=" + fourthLabelValue + ","
 251                         + fifthLabelName + "=" + fifthLabelValue + "}");
 252     }
 253
 254     private org.opendaylight.infrautils.metrics.Counter newOrExistingCounter(Object anchor, String id) {
 255         return counters.computeIfAbsent(id, newId -> {
 256             LOG.debug("New Counter metric: {}", id);
 257             return new CounterImpl(newId);
 258         });
 259     }
 260
 261     @Override
 262     public org.opendaylight.infrautils.metrics.Counter newCounter(Object anchor, String id) {
 263         requireNonNull(anchor, "anchor == null");
 264         checkForExistingID(id);
 265         return newOrExistingCounter(anchor, id);
 266     }
 267
 268     @Override
 269     public Counter newCounter(MetricDescriptor descriptor) {
 270         return newCounter(descriptor.anchor(), makeCodahaleID(descriptor));
 271     }
 272
 273     @Override
 274     public Labeled<Counter> newCounter(MetricDescriptor descriptor, String labelName) {
 275         return labelValue -> newOrExistingCounter(descriptor.anchor(),
 276                 makeCodahaleID(descriptor) + "{" + labelName + "=" + labelValue + "}");
 277     }
 278
 279     @Override
 280     public Labeled<Labeled<Counter>> newCounter(MetricDescriptor descriptor,
 281                                             String firstLabelName, String secondLabelName) {
 282         return firstLabelValue -> secondLabelValue -> newOrExistingCounter(descriptor.anchor(),
 283                 makeCodahaleID(descriptor) + "{" + firstLabelName + "=" + firstLabelValue + ","
 284                     + secondLabelName + "=" + secondLabelValue + "}");
 285     }
 286
 287     @Override
 288     public Labeled<Labeled<Labeled<Counter>>> newCounter(MetricDescriptor descriptor, String firstLabelName,
 289                                                          String secondLabelName, String thirdLabelName) {
 290         return firstLabelValue -> secondLabelValue -> thirdLabelValue ->
 291                 newOrExistingCounter(descriptor.anchor(), makeCodahaleID(descriptor) + "{"
 292                         + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue  + ","
 293                         + thirdLabelName + "=" + thirdLabelValue + "}");
 294     }
 295
 296     @Override
 297     public Labeled<Labeled<Labeled<Labeled<Counter>>>> newCounter(MetricDescriptor descriptor,
 298                                                          String firstLabelName, String secondLabelName,
 299                                                          String thirdLabelName, String fourthLabelName) {
 300         return firstLabelValue -> secondLabelValue -> thirdLabelValue -> fourthLabelValue ->
 301                 newOrExistingCounter(descriptor.anchor(), makeCodahaleID(descriptor) + "{"
 302                         + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue  + ","
 303                         + thirdLabelName + "=" + thirdLabelValue + ","
 304                         + fourthLabelName + "=" + fourthLabelValue + "}");
 305     }
 306
 307     @Override
 308     public Labeled<Labeled<Labeled<Labeled<Labeled<Counter>>>>> newCounter(MetricDescriptor descriptor,
 309                                                                   String firstLabelName, String secondLabelName,
 310                                                                   String thirdLabelName, String fourthLabelName,
 311                                                                   String fifthLabelName) {
 312         return firstLabelValue -> secondLabelValue -> thirdLabelValue -> fourthLabelValue -> fifthLabelValue ->
 313                 newOrExistingCounter(descriptor.anchor(), makeCodahaleID(descriptor) + "{"
 314                         + firstLabelName + "=" + firstLabelValue + "," + secondLabelName + "=" + secondLabelValue  + ","
 315                         + thirdLabelName + "=" + thirdLabelValue + "," + fourthLabelName + "=" + fourthLabelValue + ","
 316                         + fifthLabelName + "=" + fifthLabelValue + "}");
 317     }
 318
 319     private org.opendaylight.infrautils.metrics.Timer newOrExistingTimer(Object anchor, String id) {
 320         return timers.computeIfAbsent(id, newId -> {
 321             LOG.debug("New Timer metric: {}", id);
 322             return new TimerImpl(newId);
 323         });
 324     }
 325
 326     @Override
 327     public org.opendaylight.infrautils.metrics.Timer newTimer(Object anchor, String id) {
 328         requireNonNull(anchor, "anchor == null");
 329         checkForExistingID(id);
 330         return newOrExistingTimer(anchor, id);
 331     }
 332
 333     @Override
 334     public Timer newTimer(MetricDescriptor descriptor) {
 335         return newOrExistingTimer(descriptor.anchor(), makeCodahaleID(descriptor));
 336     }
 337
 338     @Override
 339     public Labeled<Timer> newTimer(MetricDescriptor descriptor, String labelName) {
 340         return labelValue -> newOrExistingTimer(descriptor.anchor(),
 341                 makeCodahaleID(descriptor) + "{" + labelName + "=" + labelValue + "}");
 342     }
 343
 344     @Override
 345     public Labeled<Labeled<Timer>> newTimer(MetricDescriptor descriptor,
 346                                                 String firstLabelName, String secondLabelName) {
 347         return firstLabelValue -> secondLabelValue -> newOrExistingTimer(descriptor.anchor(),
 348                 makeCodahaleID(descriptor) + "{" + firstLabelName + "=" + firstLabelValue + ","
 349                         + secondLabelName + "=" + secondLabelValue + "}");
 350     }
 351
 352     private static String makeCodahaleID(MetricDescriptor descriptor) {
 353         // We're ignoring descriptor.description() because Codahale Dropwizard Metrics
 354         // doesn't have it but other future metrics API implementations (e.g. Prometheus.io), or a
 355         // possible future metrics:list kind of CLI here, will use it.
 356         return MetricRegistry.name(descriptor.project(), descriptor.module(), descriptor.id());
 357     }
 358
 359     private void checkForExistingID(String id) {
 360         requireNonNull(id, "id == null");
 361         if (registry.getNames().contains(id)) {
 362             throw new IllegalArgumentException("Metric ID already used: " + id);
 363         }
 364     }
 365
 366     private abstract class CloseableMetricImpl implements UncheckedCloseable {
 367         private volatile boolean isClosed = false;
 368         protected final String id;
 369
 370         CloseableMetricImpl(String id) {
 371             this.id = id;
 372         }
 373
 374         protected void checkIfClosed() {
 375             if (isClosed) {
 376                 throw new IllegalStateException("Metric closed: " + id);
 377             }
 378         }
 379
 380         @Override
 381         public void close() {
 382             checkIfClosed();
 383             isClosed = true;
 384             if (!registry.remove(id)) {
 385                 LOG.warn("Metric remove did not actualy remove: {}", id);
 386             }
 387         }
 388     }
 389
 390     private final class MeterImpl extends CloseableMetricImpl implements org.opendaylight.infrautils.metrics.Meter {
 391
 392         private final com.codahale.metrics.Meter meter;
 393
 394         MeterImpl(String id) {
 395             super(id);
 396             this.meter = registry.meter(id);
 397         }
 398
 399         @Override
 400         public void mark() {
 401             checkIfClosed();
 402             meter.mark();
 403         }
 404
 405         @Override
 406         public void mark(long howMany) {
 407             checkIfClosed();
 408             meter.mark(howMany);
 409         }
 410
 411         @Override
 412         public long get() {
 413             return meter.getCount();
 414         }
 415
 416         @Override
 417         public void close() {
 418             super.close();
 419             meters.remove(id);
 420         }
 421     }
 422
 423     private final class CounterImpl extends CloseableMetricImpl
 424             implements org.opendaylight.infrautils.metrics.Counter {
 425
 426         private final com.codahale.metrics.Counter counter;
 427
 428         CounterImpl(String id) {
 429             super(id);
 430             this.counter = registry.counter(id);
 431         }
 432
 433         @Override
 434         public void increment() {
 435             checkIfClosed();
 436             counter.inc();
 437         }
 438
 439         @Override
 440         public void increment(long howMany) {
 441             checkIfClosed();
 442             counter.inc(howMany);
 443         }
 444
 445         @Override
 446         public void decrement() {
 447             checkIfClosed();
 448             counter.dec();
 449         }
 450
 451         @Override
 452         public void decrement(long howMany) {
 453             checkIfClosed();
 454             counter.dec(howMany);
 455         }
 456
 457         @Override
 458         public long get() {
 459             return counter.getCount();
 460         }
 461
 462         @Override
 463         public void close() {
 464             super.close();
 465             counters.remove(id);
 466         }
 467     }
 468
 469     private class TimerImpl extends CloseableMetricImpl implements org.opendaylight.infrautils.metrics.Timer {
 470
 471         private final com.codahale.metrics.Timer timer;
 472
 473         TimerImpl(String id) {
 474             super(id);
 475             this.timer = registry.timer(id);
 476         }
 477
 478         @Override
 479         @SuppressWarnings({ "checkstyle:IllegalCatch", "unchecked" })
 480         public <T, E extends Exception> T time(CheckedCallable<T, E> event) throws E {
 481             checkIfClosed();
 482             try {
 483                 return timer.time(event::call);
 484             } catch (Exception e) {
 485                 throw (E) e;
 486             }
 487         }
 488
 489         @Override
 490         @SuppressWarnings({ "checkstyle:IllegalCatch", "checkstyle:AvoidHidingCauseException", "unchecked" })
 491         @SuppressFBWarnings("BC_UNCONFIRMED_CAST_OF_RETURN_VALUE") // getCause() will be Exception not Throwable
 492         public <E extends Exception> void time(CheckedRunnable<E> event) throws E {
 493             checkIfClosed();
 494             try {
 495                 timer.time(() -> {
 496                     try {
 497                         event.run();
 498                     } catch (Exception exception) {
 499                         throw new InternalRuntimeException(exception);
 500                     }
 501                 });
 502             } catch (InternalRuntimeException e) {
 503                 throw (E) e.getCause();
 504             }
 505         }
 506     }
 507
 508     private static class InternalRuntimeException extends RuntimeException {
 509         private static final long serialVersionUID = 1L;
 510
 511         InternalRuntimeException(Exception exception) {
 512             super(exception);
 513         }
 514     }
 515
 516 }