From: Tom Pantelis Date: Wed, 25 Mar 2015 19:25:48 +0000 (-0400) Subject: Adjust Tx rate limiter for unused transactions X-Git-Tag: release/lithium~328^2 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=08dd5c2c443ff53f56af88a0e8dc8f34e36d2245;hp=75be58eb52cbb3846814d536f16d7829e26b95d8 Adjust Tx rate limiter for unused transactions I have a test that submits an arbitrary number of config transactions and I noticed it can take an unusually long time depending on the initial Tx rate limit setting. With the default setting of 100 it was very slow even though I could see the adjusted limit increasing where the elapsed time should've been much shorter. It turns out the config datastore rate limit wasn't issue. When a front-end Tx is created, a Tx instance is created in each data store. The operation Tx's were unused but they artificially limited the config rate since the operational rate wasn't getting updated, ie it stayed at 100. I made changes to adjust the rate limit for unused Tx's if there have been no prior "real" Tx's for that data store. In this case, the percentile metrics will be 0 so I basically adjust to the current rate from the other data store. I think this makes sense if we have no other data to go by. This required some infa changes so the ActorContext could get access to the metrics timer for the other data store. Mainly, we need a global MericsRegistry and JmxReporter across both data stores. I reused the MetricsReporter class for this to get the static instance (with changes to support multiple MetricsReporters per domain name). I also added a static method to get all the data store names. I found it useful to see the current rate limit via JMX so I added a new DatastoreInfoMXBean to report it. Change-Id: I09def94e40a1fe57779e76763e48696140f3a125 Signed-off-by: Tom Pantelis --- diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteredBoundedMailbox.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteredBoundedMailbox.java index 9b4560c726..2a6aac4d79 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteredBoundedMailbox.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteredBoundedMailbox.java @@ -26,9 +26,9 @@ public class MeteredBoundedMailbox implements MailboxType, ProducesMessageQueue< private final Logger LOG = LoggerFactory.getLogger(MeteredBoundedMailbox.class); private MeteredMessageQueue queue; - private Integer capacity; - private FiniteDuration pushTimeOut; - private MetricRegistry registry; + private final Integer capacity; + private final FiniteDuration pushTimeOut; + private final MetricRegistry registry; private final String QUEUE_SIZE = "q-size"; @@ -38,7 +38,7 @@ public class MeteredBoundedMailbox implements MailboxType, ProducesMessageQueue< this.capacity = commonConfig.getMailBoxCapacity(); this.pushTimeOut = commonConfig.getMailBoxPushTimeout(); - MetricsReporter reporter = MetricsReporter.getInstance(); + MetricsReporter reporter = MetricsReporter.getInstance(MeteringBehavior.DOMAIN); registry = reporter.getMetricsRegistry(); } @@ -58,7 +58,9 @@ public class MeteredBoundedMailbox implements MailboxType, ProducesMessageQueue< String metricName = MetricRegistry.name(actorName, QUEUE_SIZE); if (registry.getMetrics().containsKey(metricName)) + { return; //already registered + } Gauge queueSize = getQueueSizeGuage(monitoredQueue); registerQueueSizeMetric(metricName, queueSize); diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteringBehavior.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteringBehavior.java index 9ff185a61e..c04e2e9340 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteringBehavior.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteringBehavior.java @@ -26,10 +26,11 @@ import org.opendaylight.controller.cluster.reporting.MetricsReporter; * The information is reported to {@link org.opendaylight.controller.cluster.reporting.MetricsReporter} */ public class MeteringBehavior implements Procedure { + public static final String DOMAIN = "org.opendaylight.controller.actor.metric"; private final UntypedActor meteredActor; - private final MetricRegistry METRICREGISTRY = MetricsReporter.getInstance().getMetricsRegistry(); + private final MetricRegistry METRICREGISTRY = MetricsReporter.getInstance(DOMAIN).getMetricsRegistry(); private final String MSG_PROCESSING_RATE = "msg-rate"; private String actorQualifiedName; diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/reporting/MetricsReporter.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/reporting/MetricsReporter.java index b400fcab7d..9b93744368 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/reporting/MetricsReporter.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/reporting/MetricsReporter.java @@ -9,6 +9,9 @@ package org.opendaylight.controller.cluster.reporting; import com.codahale.metrics.JmxReporter; import com.codahale.metrics.MetricRegistry; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; /** * Maintains metrics registry that is provided to reporters. @@ -20,26 +23,36 @@ import com.codahale.metrics.MetricRegistry; */ public class MetricsReporter implements AutoCloseable { - private static final MetricRegistry METRICS_REGISTRY = new MetricRegistry(); - private static final String DOMAIN = "org.opendaylight.controller.actor.metric"; - private static final MetricsReporter INSTANCE = new MetricsReporter(); - - private final JmxReporter jmxReporter = JmxReporter.forRegistry(METRICS_REGISTRY).inDomain(DOMAIN).build(); - - private MetricsReporter() { + private static LoadingCache METRIC_REPORTERS = CacheBuilder.newBuilder().build( + new CacheLoader() { + @Override + public MetricsReporter load(String domainName) { + return new MetricsReporter(domainName); + } + }); + + private final String domainName; + private final JmxReporter jmxReporter; + private final MetricRegistry metricRegistry = new MetricRegistry(); + + private MetricsReporter(String domainName) { + this.domainName = domainName; + jmxReporter = JmxReporter.forRegistry(metricRegistry).inDomain(domainName).build(); jmxReporter.start(); } - public static MetricsReporter getInstance() { - return INSTANCE; + public static MetricsReporter getInstance(String domainName) { + return METRIC_REPORTERS.getUnchecked(domainName); } public MetricRegistry getMetricsRegistry() { - return METRICS_REGISTRY; + return metricRegistry; } @Override public void close() { jmxReporter.close(); + + METRIC_REPORTERS.invalidate(domainName); } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java index d5142c94a6..c3cc499865 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java @@ -9,6 +9,8 @@ package org.opendaylight.controller.cluster.datastore; import akka.util.Timeout; +import com.google.common.collect.Sets; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.text.WordUtils; import org.opendaylight.controller.cluster.datastore.config.ConfigurationReader; @@ -25,6 +27,7 @@ import scala.concurrent.duration.FiniteDuration; * @author Thomas Pantelis */ public class DatastoreContext { + public static final String METRICS_DOMAIN = "org.opendaylight.controller.cluster.datastore"; public static final Duration DEFAULT_SHARD_TRANSACTION_IDLE_TIMEOUT = Duration.create(10, TimeUnit.MINUTES); public static final int DEFAULT_OPERATION_TIMEOUT_IN_SECONDS = 5; @@ -44,6 +47,8 @@ public class DatastoreContext { public static final String UNKNOWN_DATA_STORE_TYPE = "unknown"; public static final int DEFAULT_SHARD_BATCHED_MODIFICATION_COUNT= 100; + private static Set globalDatastoreTypes = Sets.newConcurrentHashSet(); + private InMemoryDOMDataStoreConfigProperties dataStoreProperties; private Duration shardTransactionIdleTimeout = DatastoreContext.DEFAULT_SHARD_TRANSACTION_IDLE_TIMEOUT; private int operationTimeoutInSeconds = DEFAULT_OPERATION_TIMEOUT_IN_SECONDS; @@ -60,6 +65,10 @@ public class DatastoreContext { private int shardBatchedModificationCount = DEFAULT_SHARD_BATCHED_MODIFICATION_COUNT; private boolean writeOnlyTransactionOptimizationsEnabled = false; + public static Set getGlobalDatastoreTypes() { + return globalDatastoreTypes; + } + private DatastoreContext() { setShardJournalRecoveryLogBatchSize(DEFAULT_JOURNAL_RECOVERY_BATCH_SIZE); setSnapshotBatchCount(DEFAULT_SNAPSHOT_BATCH_COUNT); @@ -361,6 +370,11 @@ public class DatastoreContext { datastoreContext.dataStoreProperties = InMemoryDOMDataStoreConfigProperties.create( maxShardDataChangeExecutorPoolSize, maxShardDataChangeExecutorQueueSize, maxShardDataChangeListenerQueueSize, maxShardDataStoreExecutorQueueSize); + + if(datastoreContext.dataStoreType != null) { + globalDatastoreTypes.add(datastoreContext.dataStoreType); + } + return datastoreContext; } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DistributedDataStore.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DistributedDataStore.java index 5ade98cb86..c79de94567 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DistributedDataStore.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DistributedDataStore.java @@ -15,6 +15,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier; import org.opendaylight.controller.cluster.datastore.jmx.mbeans.DatastoreConfigurationMXBeanImpl; +import org.opendaylight.controller.cluster.datastore.jmx.mbeans.DatastoreInfoMXBeanImpl; import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategyFactory; import org.opendaylight.controller.cluster.datastore.utils.ActorContext; import org.opendaylight.controller.cluster.datastore.utils.Dispatchers; @@ -52,7 +53,9 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener, private DatastoreConfigurationMXBeanImpl datastoreConfigMXBean; - private CountDownLatch waitTillReadyCountDownLatch = new CountDownLatch(1); + private DatastoreInfoMXBeanImpl datastoreInfoMXBean; + + private final CountDownLatch waitTillReadyCountDownLatch = new CountDownLatch(1); private final String type; @@ -84,6 +87,9 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener, datastoreConfigMXBean = new DatastoreConfigurationMXBeanImpl(datastoreContext.getDataStoreMXBeanType()); datastoreConfigMXBean.setContext(datastoreContext); datastoreConfigMXBean.registerMBean(); + + datastoreInfoMXBean = new DatastoreInfoMXBeanImpl(datastoreContext.getDataStoreMXBeanType(), actorContext); + datastoreInfoMXBean.registerMBean(); } public DistributedDataStore(ActorContext actorContext) { @@ -157,6 +163,7 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener, @Override public void close() { datastoreConfigMXBean.unregisterMBean(); + datastoreInfoMXBean.unregisterMBean(); if(closeable != null) { try { diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java index 64f914b19f..f1ba4eabb9 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java @@ -393,6 +393,7 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { if(txFutureCallbackMap.size() == 0) { onTransactionReady(Collections.>emptyList()); + TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext); return NoOpDOMStoreThreePhaseCommitCohort.INSTANCE; } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCallback.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCallback.java index 1202a909d5..526ce59aab 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCallback.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCallback.java @@ -44,11 +44,30 @@ public class TransactionRateLimitingCallback implements OperationCallback{ Preconditions.checkState(timerContext != null, "Call run before success"); timerContext.stop(); + double newRateLimit = calculateNewRateLimit(commitTimer, actorContext.getDatastoreContext()); + + LOG.debug("Data Store {} commit rateLimit adjusted to {}", actorContext.getDataStoreType(), newRateLimit); + + actorContext.setTxCreationLimit(newRateLimit); + } + + @Override + public void failure() { + // This would mean we couldn't get a transaction completed in 30 seconds which is + // the default transaction commit timeout. Using the timeout information to figure out the rate limit is + // not going to be useful - so we leave it as it is + } + + private static double calculateNewRateLimit(Timer commitTimer, DatastoreContext context) { + if(commitTimer == null) { + // This can happen in unit tests. + return 0; + } + Snapshot timerSnapshot = commitTimer.getSnapshot(); double newRateLimit = 0; - long commitTimeoutInSeconds = actorContext.getDatastoreContext() - .getShardTransactionCommitTimeoutInSeconds(); + long commitTimeoutInSeconds = context.getShardTransactionCommitTimeoutInSeconds(); long commitTimeoutInNanos = TimeUnit.SECONDS.toNanos(commitTimeoutInSeconds); // Find the time that it takes for transactions to get executed in every 10th percentile @@ -59,7 +78,7 @@ public class TransactionRateLimitingCallback implements OperationCallback{ if(percentileTimeInNanos > 0) { // Figure out the rate limit for the i*10th percentile in nanos - double percentileRateLimit = ((double) commitTimeoutInNanos / percentileTimeInNanos); + double percentileRateLimit = (commitTimeoutInNanos / percentileTimeInNanos); // Add the percentileRateLimit to the total rate limit newRateLimit += percentileRateLimit; @@ -67,17 +86,38 @@ public class TransactionRateLimitingCallback implements OperationCallback{ } // Compute the rate limit per second - newRateLimit = newRateLimit/(commitTimeoutInSeconds*10); - - LOG.debug("Data Store {} commit rateLimit adjusted to {}", actorContext.getDataStoreType(), newRateLimit); - - actorContext.setTxCreationLimit(newRateLimit); + return newRateLimit/(commitTimeoutInSeconds*10); } - @Override - public void failure() { - // This would mean we couldn't get a transaction completed in 30 seconds which is - // the default transaction commit timeout. Using the timeout information to figure out the rate limit is - // not going to be useful - so we leave it as it is + public static void adjustRateLimitForUnusedTransaction(ActorContext actorContext) { + // Unused transactions in one data store can artificially limit the rate for other data stores + // if the first data store's rate is still at a lower initial rate since the front-end creates + // transactions in each data store up-front even though the client may not actually submit changes. + // So we may have to adjust the rate for data stores with unused transactions. + + // First calculate the current rate for the data store. If it's 0 then there have been no + // actual transactions committed to the data store. + + double newRateLimit = calculateNewRateLimit(actorContext.getOperationTimer(COMMIT), + actorContext.getDatastoreContext()); + if(newRateLimit == 0.0) { + // Since we have no rate data for unused Tx's data store, adjust to the rate from another + // data store that does have rate data. + for(String datastoreType: DatastoreContext.getGlobalDatastoreTypes()) { + if(datastoreType.equals(actorContext.getDataStoreType())) { + continue; + } + + newRateLimit = calculateNewRateLimit(actorContext.getOperationTimer(datastoreType, COMMIT), + actorContext.getDatastoreContext()); + if(newRateLimit > 0.0) { + LOG.debug("On unused Tx - data Store {} commit rateLimit adjusted to {}", + actorContext.getDataStoreType(), newRateLimit); + + actorContext.setTxCreationLimit(newRateLimit); + break; + } + } + } } } \ No newline at end of file diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBean.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBean.java new file mode 100644 index 0000000000..d393d64601 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBean.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.jmx.mbeans; + +/** + * JMX bean for general datastore info. + * + * @author Thomas Pantelis + */ +public interface DatastoreInfoMXBean { + double getTransactionCreationRateLimit(); +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBeanImpl.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBeanImpl.java new file mode 100644 index 0000000000..7976344c7c --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBeanImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.jmx.mbeans; + +import org.opendaylight.controller.cluster.datastore.utils.ActorContext; +import org.opendaylight.controller.md.sal.common.util.jmx.AbstractMXBean; + +/** + * Implementation of DatastoreInfoMXBean. + * + * @author Thomas Pantelis + */ +public class DatastoreInfoMXBeanImpl extends AbstractMXBean implements DatastoreInfoMXBean { + + private final ActorContext actorContext; + + public DatastoreInfoMXBeanImpl(String mxBeanType, ActorContext actorContext) { + super("GeneralRuntimeInfo", mxBeanType, null); + this.actorContext = actorContext; + } + + + @Override + public double getTransactionCreationRateLimit() { + return actorContext.getTxCreationLimit(); + } +} diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/ActorContext.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/ActorContext.java index b6250fc1cc..f53368d886 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/ActorContext.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/ActorContext.java @@ -20,7 +20,6 @@ import akka.dispatch.Mapper; import akka.dispatch.OnComplete; import akka.pattern.AskTimeoutException; import akka.util.Timeout; -import com.codahale.metrics.JmxReporter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; @@ -47,6 +46,7 @@ import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound; import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound; import org.opendaylight.controller.cluster.datastore.messages.PrimaryFound; import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext; +import org.opendaylight.controller.cluster.reporting.MetricsReporter; import org.opendaylight.yangtools.yang.model.api.SchemaContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,10 +64,8 @@ import scala.concurrent.duration.FiniteDuration; */ public class ActorContext { private static final Logger LOG = LoggerFactory.getLogger(ActorContext.class); - private static final String UNKNOWN_DATA_STORE_TYPE = "unknown"; private static final String DISTRIBUTED_DATA_STORE_METRIC_REGISTRY = "distributed-data-store"; private static final String METRIC_RATE = "rate"; - private static final String DOMAIN = "org.opendaylight.controller.cluster.datastore"; private static final Mapper FIND_PRIMARY_FAILURE_TRANSFORMER = new Mapper() { @Override @@ -94,8 +92,6 @@ public class ActorContext { private Timeout operationTimeout; private final String selfAddressHostPort; private RateLimiter txRateLimiter; - private final MetricRegistry metricRegistry = new MetricRegistry(); - private final JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry).inDomain(DOMAIN).build(); private final int transactionOutstandingOperationLimit; private Timeout transactionCommitOperationTimeout; private Timeout shardInitializationTimeout; @@ -104,6 +100,7 @@ public class ActorContext { private volatile SchemaContext schemaContext; private volatile boolean updated; + private final MetricRegistry metricRegistry = MetricsReporter.getInstance(DatastoreContext.METRICS_DOMAIN).getMetricsRegistry(); public ActorContext(ActorSystem actorSystem, ActorRef shardManager, ClusterWrapper clusterWrapper, Configuration configuration) { @@ -131,8 +128,6 @@ public class ActorContext { } transactionOutstandingOperationLimit = new CommonConfig(this.getActorSystem().settings().config()).getMailBoxCapacity(); - jmxReporter.start(); - } private void setCachedProperties() { @@ -482,7 +477,12 @@ public class ActorContext { * @return */ public Timer getOperationTimer(String operationName){ - final String rate = MetricRegistry.name(DISTRIBUTED_DATA_STORE_METRIC_REGISTRY, datastoreContext.getDataStoreType(), operationName, METRIC_RATE); + return getOperationTimer(datastoreContext.getDataStoreType(), operationName); + } + + public Timer getOperationTimer(String dataStoreType, String operationName){ + final String rate = MetricRegistry.name(DISTRIBUTED_DATA_STORE_METRIC_REGISTRY, dataStoreType, + operationName, METRIC_RATE); return metricRegistry.timer(rate); } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCommitCallbackTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCommitCallbackTest.java index f97e325472..69a023ad31 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCommitCallbackTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCommitCallbackTest.java @@ -23,6 +23,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Matchers; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.opendaylight.controller.cluster.datastore.utils.ActorContext; @@ -170,12 +171,48 @@ public class TransactionRateLimitingCommitCallbackTest { } + @Test + public void testAdjustRateLimitForUnusedTransaction() { + doReturn(commitTimer).when(actorContext).getOperationTimer("one", "commit"); + doReturn("one").when(actorContext).getDataStoreType(); + + Timer commitTimer2 = Mockito.mock(Timer.class); + Snapshot commitSnapshot2 = Mockito.mock(Snapshot.class); + + doReturn(commitSnapshot2).when(commitTimer2).getSnapshot(); + + doReturn(commitTimer2).when(actorContext).getOperationTimer("two", "commit"); + + DatastoreContext.newBuilder().dataStoreType("one").build(); + DatastoreContext.newBuilder().dataStoreType("two").build(); + + doReturn(TimeUnit.MICROSECONDS.toNanos(500) * 1D).when(commitSnapshot).getValue(1 * 0.1); + + TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext); + + verify(actorContext, never()).setTxCreationLimit(anyDouble()); + + Mockito.reset(commitSnapshot); + + TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext); + + verify(actorContext, never()).setTxCreationLimit(anyDouble()); + + System.out.println(""+TimeUnit.SECONDS.toNanos(30)/TimeUnit.MICROSECONDS.toNanos(100)); + + doReturn(TimeUnit.MICROSECONDS.toNanos(100) * 1D).when(commitSnapshot2).getValue(1 * 0.1); + + TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext); + + verify(actorContext).setTxCreationLimit(Matchers.doubleThat(approximately(1000))); + } + public Matcher approximately(final double val){ return new BaseMatcher() { @Override public boolean matches(Object o) { Double aDouble = (Double) o; - return aDouble > val && aDouble < val+1; + return aDouble >= val && aDouble <= val+1; } @Override