Adjust Tx rate limiter for unused transactions 82/17182/4
authorTom Pantelis <tpanteli@brocade.com>
Wed, 25 Mar 2015 19:25:48 +0000 (15:25 -0400)
committerTom Pantelis <tpanteli@brocade.com>
Sun, 29 Mar 2015 06:53:46 +0000 (02:53 -0400)
I have a test that submits an arbitrary number of config transactions
and I noticed it can take an unusually long time depending on the
initial Tx rate limit setting. With the default setting of 100 it was
very slow even though I could see the adjusted limit increasing where
the elapsed time should've been much shorter.

It turns out the config datastore rate limit wasn't issue. When a
front-end Tx is created, a Tx instance is created in each data store.
The operation Tx's were unused but they artificially limited the config
rate since the operational rate wasn't getting updated, ie it stayed at
100.

I made changes to adjust the rate limit for unused Tx's if there have
been no prior "real" Tx's for that data store. In this case, the
percentile metrics will be 0 so I basically adjust to the current rate
from the other data store. I think this makes sense if we have no other
data to go by.

This required some infa changes so the ActorContext could get access to
the metrics timer for the other data store. Mainly, we need a global
MericsRegistry and JmxReporter across both data stores. I reused the
MetricsReporter class for this to get the static instance (with changes to
support multiple MetricsReporters per domain name). I also added
a static method to get all the data store names.

I found it useful to see the current rate limit via JMX so I added a new
DatastoreInfoMXBean to report it.

Change-Id: I09def94e40a1fe57779e76763e48696140f3a125
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>
opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteredBoundedMailbox.java
opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/common/actor/MeteringBehavior.java
opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/reporting/MetricsReporter.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DistributedDataStore.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCallback.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBean.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBeanImpl.java [new file with mode: 0644]
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/utils/ActorContext.java
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/TransactionRateLimitingCommitCallbackTest.java

index 9b4560c..2a6aac4 100644 (file)
@@ -26,9 +26,9 @@ public class MeteredBoundedMailbox implements MailboxType, ProducesMessageQueue<
     private final Logger LOG = LoggerFactory.getLogger(MeteredBoundedMailbox.class);
 
     private MeteredMessageQueue queue;
-    private Integer capacity;
-    private FiniteDuration pushTimeOut;
-    private MetricRegistry registry;
+    private final Integer capacity;
+    private final FiniteDuration pushTimeOut;
+    private final MetricRegistry registry;
 
     private final String QUEUE_SIZE = "q-size";
 
@@ -38,7 +38,7 @@ public class MeteredBoundedMailbox implements MailboxType, ProducesMessageQueue<
         this.capacity = commonConfig.getMailBoxCapacity();
         this.pushTimeOut = commonConfig.getMailBoxPushTimeout();
 
-        MetricsReporter reporter = MetricsReporter.getInstance();
+        MetricsReporter reporter = MetricsReporter.getInstance(MeteringBehavior.DOMAIN);
         registry = reporter.getMetricsRegistry();
     }
 
@@ -58,7 +58,9 @@ public class MeteredBoundedMailbox implements MailboxType, ProducesMessageQueue<
         String metricName = MetricRegistry.name(actorName, QUEUE_SIZE);
 
         if (registry.getMetrics().containsKey(metricName))
+        {
             return; //already registered
+        }
 
         Gauge<Integer> queueSize = getQueueSizeGuage(monitoredQueue);
         registerQueueSizeMetric(metricName, queueSize);
index 9ff185a..c04e2e9 100644 (file)
@@ -26,10 +26,11 @@ import org.opendaylight.controller.cluster.reporting.MetricsReporter;
  * The information is reported to {@link org.opendaylight.controller.cluster.reporting.MetricsReporter}
  */
 public class MeteringBehavior implements Procedure<Object> {
+    public static final String DOMAIN = "org.opendaylight.controller.actor.metric";
 
     private final UntypedActor meteredActor;
 
-    private final MetricRegistry METRICREGISTRY = MetricsReporter.getInstance().getMetricsRegistry();
+    private final MetricRegistry METRICREGISTRY = MetricsReporter.getInstance(DOMAIN).getMetricsRegistry();
     private final String MSG_PROCESSING_RATE = "msg-rate";
 
     private String actorQualifiedName;
index b400fca..9b93744 100644 (file)
@@ -9,6 +9,9 @@ package org.opendaylight.controller.cluster.reporting;
 
 import com.codahale.metrics.JmxReporter;
 import com.codahale.metrics.MetricRegistry;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
 
 /**
  * Maintains metrics registry that is provided to reporters.
@@ -20,26 +23,36 @@ import com.codahale.metrics.MetricRegistry;
  */
 public class MetricsReporter implements AutoCloseable {
 
-    private static final MetricRegistry METRICS_REGISTRY = new MetricRegistry();
-    private static final String DOMAIN = "org.opendaylight.controller.actor.metric";
-    private static final MetricsReporter INSTANCE = new MetricsReporter();
-
-    private final JmxReporter jmxReporter = JmxReporter.forRegistry(METRICS_REGISTRY).inDomain(DOMAIN).build();
-
-    private MetricsReporter() {
+    private static LoadingCache<String, MetricsReporter> METRIC_REPORTERS = CacheBuilder.newBuilder().build(
+            new CacheLoader<String, MetricsReporter>() {
+                @Override
+                public MetricsReporter load(String domainName) {
+                    return new MetricsReporter(domainName);
+                }
+            });
+
+    private final String domainName;
+    private final JmxReporter jmxReporter;
+    private final MetricRegistry metricRegistry = new MetricRegistry();
+
+    private MetricsReporter(String domainName) {
+        this.domainName = domainName;
+        jmxReporter = JmxReporter.forRegistry(metricRegistry).inDomain(domainName).build();
         jmxReporter.start();
     }
 
-    public static MetricsReporter getInstance() {
-        return INSTANCE;
+    public static MetricsReporter getInstance(String domainName) {
+        return METRIC_REPORTERS.getUnchecked(domainName);
     }
 
     public MetricRegistry getMetricsRegistry() {
-        return METRICS_REGISTRY;
+        return metricRegistry;
     }
 
     @Override
     public void close() {
         jmxReporter.close();
+
+        METRIC_REPORTERS.invalidate(domainName);
     }
 }
index d5142c9..c3cc499 100644 (file)
@@ -9,6 +9,8 @@
 package org.opendaylight.controller.cluster.datastore;
 
 import akka.util.Timeout;
+import com.google.common.collect.Sets;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.lang3.text.WordUtils;
 import org.opendaylight.controller.cluster.datastore.config.ConfigurationReader;
@@ -25,6 +27,7 @@ import scala.concurrent.duration.FiniteDuration;
  * @author Thomas Pantelis
  */
 public class DatastoreContext {
+    public static final String METRICS_DOMAIN = "org.opendaylight.controller.cluster.datastore";
 
     public static final Duration DEFAULT_SHARD_TRANSACTION_IDLE_TIMEOUT = Duration.create(10, TimeUnit.MINUTES);
     public static final int DEFAULT_OPERATION_TIMEOUT_IN_SECONDS = 5;
@@ -44,6 +47,8 @@ public class DatastoreContext {
     public static final String UNKNOWN_DATA_STORE_TYPE = "unknown";
     public static final int DEFAULT_SHARD_BATCHED_MODIFICATION_COUNT= 100;
 
+    private static Set<String> globalDatastoreTypes = Sets.newConcurrentHashSet();
+
     private InMemoryDOMDataStoreConfigProperties dataStoreProperties;
     private Duration shardTransactionIdleTimeout = DatastoreContext.DEFAULT_SHARD_TRANSACTION_IDLE_TIMEOUT;
     private int operationTimeoutInSeconds = DEFAULT_OPERATION_TIMEOUT_IN_SECONDS;
@@ -60,6 +65,10 @@ public class DatastoreContext {
     private int shardBatchedModificationCount = DEFAULT_SHARD_BATCHED_MODIFICATION_COUNT;
     private boolean writeOnlyTransactionOptimizationsEnabled = false;
 
+    public static Set<String> getGlobalDatastoreTypes() {
+        return globalDatastoreTypes;
+    }
+
     private DatastoreContext() {
         setShardJournalRecoveryLogBatchSize(DEFAULT_JOURNAL_RECOVERY_BATCH_SIZE);
         setSnapshotBatchCount(DEFAULT_SNAPSHOT_BATCH_COUNT);
@@ -361,6 +370,11 @@ public class DatastoreContext {
             datastoreContext.dataStoreProperties = InMemoryDOMDataStoreConfigProperties.create(
                     maxShardDataChangeExecutorPoolSize, maxShardDataChangeExecutorQueueSize,
                     maxShardDataChangeListenerQueueSize, maxShardDataStoreExecutorQueueSize);
+
+            if(datastoreContext.dataStoreType != null) {
+                globalDatastoreTypes.add(datastoreContext.dataStoreType);
+            }
+
             return datastoreContext;
         }
     }
index 5ade98c..c79de94 100644 (file)
@@ -15,6 +15,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
 import org.opendaylight.controller.cluster.datastore.jmx.mbeans.DatastoreConfigurationMXBeanImpl;
+import org.opendaylight.controller.cluster.datastore.jmx.mbeans.DatastoreInfoMXBeanImpl;
 import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategyFactory;
 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
 import org.opendaylight.controller.cluster.datastore.utils.Dispatchers;
@@ -52,7 +53,9 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener,
 
     private DatastoreConfigurationMXBeanImpl datastoreConfigMXBean;
 
-    private CountDownLatch waitTillReadyCountDownLatch = new CountDownLatch(1);
+    private DatastoreInfoMXBeanImpl datastoreInfoMXBean;
+
+    private final CountDownLatch waitTillReadyCountDownLatch = new CountDownLatch(1);
 
     private final String type;
 
@@ -84,6 +87,9 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener,
         datastoreConfigMXBean = new DatastoreConfigurationMXBeanImpl(datastoreContext.getDataStoreMXBeanType());
         datastoreConfigMXBean.setContext(datastoreContext);
         datastoreConfigMXBean.registerMBean();
+
+        datastoreInfoMXBean = new DatastoreInfoMXBeanImpl(datastoreContext.getDataStoreMXBeanType(), actorContext);
+        datastoreInfoMXBean.registerMBean();
     }
 
     public DistributedDataStore(ActorContext actorContext) {
@@ -157,6 +163,7 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener,
     @Override
     public void close() {
         datastoreConfigMXBean.unregisterMBean();
+        datastoreInfoMXBean.unregisterMBean();
 
         if(closeable != null) {
             try {
index 64f914b..f1ba4ea 100644 (file)
@@ -393,6 +393,7 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction {
 
         if(txFutureCallbackMap.size() == 0) {
             onTransactionReady(Collections.<Future<ActorSelection>>emptyList());
+            TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext);
             return NoOpDOMStoreThreePhaseCommitCohort.INSTANCE;
         }
 
index 1202a90..526ce59 100644 (file)
@@ -44,11 +44,30 @@ public class TransactionRateLimitingCallback implements OperationCallback{
         Preconditions.checkState(timerContext != null, "Call run before success");
         timerContext.stop();
 
+        double newRateLimit = calculateNewRateLimit(commitTimer, actorContext.getDatastoreContext());
+
+        LOG.debug("Data Store {} commit rateLimit adjusted to {}", actorContext.getDataStoreType(), newRateLimit);
+
+        actorContext.setTxCreationLimit(newRateLimit);
+    }
+
+    @Override
+    public void failure() {
+        // This would mean we couldn't get a transaction completed in 30 seconds which is
+        // the default transaction commit timeout. Using the timeout information to figure out the rate limit is
+        // not going to be useful - so we leave it as it is
+    }
+
+    private static double calculateNewRateLimit(Timer commitTimer, DatastoreContext context) {
+        if(commitTimer == null) {
+            // This can happen in unit tests.
+            return 0;
+        }
+
         Snapshot timerSnapshot = commitTimer.getSnapshot();
         double newRateLimit = 0;
 
-        long commitTimeoutInSeconds = actorContext.getDatastoreContext()
-                .getShardTransactionCommitTimeoutInSeconds();
+        long commitTimeoutInSeconds = context.getShardTransactionCommitTimeoutInSeconds();
         long commitTimeoutInNanos = TimeUnit.SECONDS.toNanos(commitTimeoutInSeconds);
 
         // Find the time that it takes for transactions to get executed in every 10th percentile
@@ -59,7 +78,7 @@ public class TransactionRateLimitingCallback implements OperationCallback{
 
             if(percentileTimeInNanos > 0) {
                 // Figure out the rate limit for the i*10th percentile in nanos
-                double percentileRateLimit = ((double) commitTimeoutInNanos / percentileTimeInNanos);
+                double percentileRateLimit = (commitTimeoutInNanos / percentileTimeInNanos);
 
                 // Add the percentileRateLimit to the total rate limit
                 newRateLimit += percentileRateLimit;
@@ -67,17 +86,38 @@ public class TransactionRateLimitingCallback implements OperationCallback{
         }
 
         // Compute the rate limit per second
-        newRateLimit = newRateLimit/(commitTimeoutInSeconds*10);
-
-        LOG.debug("Data Store {} commit rateLimit adjusted to {}", actorContext.getDataStoreType(), newRateLimit);
-
-        actorContext.setTxCreationLimit(newRateLimit);
+        return newRateLimit/(commitTimeoutInSeconds*10);
     }
 
-    @Override
-    public void failure() {
-        // This would mean we couldn't get a transaction completed in 30 seconds which is
-        // the default transaction commit timeout. Using the timeout information to figure out the rate limit is
-        // not going to be useful - so we leave it as it is
+    public static void adjustRateLimitForUnusedTransaction(ActorContext actorContext) {
+        // Unused transactions in one data store can artificially limit the rate for other data stores
+        // if the first data store's rate is still at a lower initial rate since the front-end creates
+        // transactions in each data store up-front even though the client may not actually submit changes.
+        // So we may have to adjust the rate for data stores with unused transactions.
+
+        // First calculate the current rate for the data store. If it's 0 then there have been no
+        // actual transactions committed to the data store.
+
+        double newRateLimit = calculateNewRateLimit(actorContext.getOperationTimer(COMMIT),
+                actorContext.getDatastoreContext());
+        if(newRateLimit == 0.0) {
+            // Since we have no rate data for unused Tx's data store, adjust to the rate from another
+            // data store that does have rate data.
+            for(String datastoreType: DatastoreContext.getGlobalDatastoreTypes()) {
+                if(datastoreType.equals(actorContext.getDataStoreType())) {
+                    continue;
+                }
+
+                newRateLimit = calculateNewRateLimit(actorContext.getOperationTimer(datastoreType, COMMIT),
+                        actorContext.getDatastoreContext());
+                if(newRateLimit > 0.0) {
+                    LOG.debug("On unused Tx - data Store {} commit rateLimit adjusted to {}",
+                            actorContext.getDataStoreType(), newRateLimit);
+
+                    actorContext.setTxCreationLimit(newRateLimit);
+                    break;
+                }
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBean.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBean.java
new file mode 100644 (file)
index 0000000..d393d64
--- /dev/null
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.datastore.jmx.mbeans;
+
+/**
+ * JMX bean for general datastore info.
+ *
+ * @author Thomas Pantelis
+ */
+public interface DatastoreInfoMXBean {
+    double getTransactionCreationRateLimit();
+}
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBeanImpl.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreInfoMXBeanImpl.java
new file mode 100644 (file)
index 0000000..7976344
--- /dev/null
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.datastore.jmx.mbeans;
+
+import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
+import org.opendaylight.controller.md.sal.common.util.jmx.AbstractMXBean;
+
+/**
+ * Implementation of DatastoreInfoMXBean.
+ *
+ * @author Thomas Pantelis
+ */
+public class DatastoreInfoMXBeanImpl extends AbstractMXBean implements DatastoreInfoMXBean {
+
+    private final ActorContext actorContext;
+
+    public DatastoreInfoMXBeanImpl(String mxBeanType, ActorContext actorContext) {
+        super("GeneralRuntimeInfo", mxBeanType, null);
+        this.actorContext = actorContext;
+    }
+
+
+    @Override
+    public double getTransactionCreationRateLimit() {
+        return actorContext.getTxCreationLimit();
+    }
+}
index b6250fc..f53368d 100644 (file)
@@ -20,7 +20,6 @@ import akka.dispatch.Mapper;
 import akka.dispatch.OnComplete;
 import akka.pattern.AskTimeoutException;
 import akka.util.Timeout;
-import com.codahale.metrics.JmxReporter;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Timer;
 import com.google.common.annotations.VisibleForTesting;
@@ -47,6 +46,7 @@ import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
 import org.opendaylight.controller.cluster.datastore.messages.PrimaryFound;
 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
+import org.opendaylight.controller.cluster.reporting.MetricsReporter;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -64,10 +64,8 @@ import scala.concurrent.duration.FiniteDuration;
  */
 public class ActorContext {
     private static final Logger LOG = LoggerFactory.getLogger(ActorContext.class);
-    private static final String UNKNOWN_DATA_STORE_TYPE = "unknown";
     private static final String DISTRIBUTED_DATA_STORE_METRIC_REGISTRY = "distributed-data-store";
     private static final String METRIC_RATE = "rate";
-    private static final String DOMAIN = "org.opendaylight.controller.cluster.datastore";
     private static final Mapper<Throwable, Throwable> FIND_PRIMARY_FAILURE_TRANSFORMER =
                                                               new Mapper<Throwable, Throwable>() {
         @Override
@@ -94,8 +92,6 @@ public class ActorContext {
     private Timeout operationTimeout;
     private final String selfAddressHostPort;
     private RateLimiter txRateLimiter;
-    private final MetricRegistry metricRegistry = new MetricRegistry();
-    private final JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry).inDomain(DOMAIN).build();
     private final int transactionOutstandingOperationLimit;
     private Timeout transactionCommitOperationTimeout;
     private Timeout shardInitializationTimeout;
@@ -104,6 +100,7 @@ public class ActorContext {
 
     private volatile SchemaContext schemaContext;
     private volatile boolean updated;
+    private final MetricRegistry metricRegistry = MetricsReporter.getInstance(DatastoreContext.METRICS_DOMAIN).getMetricsRegistry();
 
     public ActorContext(ActorSystem actorSystem, ActorRef shardManager,
             ClusterWrapper clusterWrapper, Configuration configuration) {
@@ -131,8 +128,6 @@ public class ActorContext {
         }
 
         transactionOutstandingOperationLimit = new CommonConfig(this.getActorSystem().settings().config()).getMailBoxCapacity();
-        jmxReporter.start();
-
     }
 
     private void setCachedProperties() {
@@ -482,7 +477,12 @@ public class ActorContext {
      * @return
      */
     public Timer getOperationTimer(String operationName){
-        final String rate = MetricRegistry.name(DISTRIBUTED_DATA_STORE_METRIC_REGISTRY, datastoreContext.getDataStoreType(), operationName, METRIC_RATE);
+        return getOperationTimer(datastoreContext.getDataStoreType(), operationName);
+    }
+
+    public Timer getOperationTimer(String dataStoreType, String operationName){
+        final String rate = MetricRegistry.name(DISTRIBUTED_DATA_STORE_METRIC_REGISTRY, dataStoreType,
+                operationName, METRIC_RATE);
         return metricRegistry.timer(rate);
     }
 
index f97e325..69a023a 100644 (file)
@@ -23,6 +23,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Matchers;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
 
@@ -170,12 +171,48 @@ public class TransactionRateLimitingCommitCallbackTest {
 
     }
 
+    @Test
+    public void testAdjustRateLimitForUnusedTransaction() {
+        doReturn(commitTimer).when(actorContext).getOperationTimer("one", "commit");
+        doReturn("one").when(actorContext).getDataStoreType();
+
+        Timer commitTimer2 = Mockito.mock(Timer.class);
+        Snapshot commitSnapshot2 = Mockito.mock(Snapshot.class);
+
+        doReturn(commitSnapshot2).when(commitTimer2).getSnapshot();
+
+        doReturn(commitTimer2).when(actorContext).getOperationTimer("two", "commit");
+
+        DatastoreContext.newBuilder().dataStoreType("one").build();
+        DatastoreContext.newBuilder().dataStoreType("two").build();
+
+        doReturn(TimeUnit.MICROSECONDS.toNanos(500) * 1D).when(commitSnapshot).getValue(1 * 0.1);
+
+        TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext);
+
+        verify(actorContext, never()).setTxCreationLimit(anyDouble());
+
+        Mockito.reset(commitSnapshot);
+
+        TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext);
+
+        verify(actorContext, never()).setTxCreationLimit(anyDouble());
+
+        System.out.println(""+TimeUnit.SECONDS.toNanos(30)/TimeUnit.MICROSECONDS.toNanos(100));
+
+        doReturn(TimeUnit.MICROSECONDS.toNanos(100) * 1D).when(commitSnapshot2).getValue(1 * 0.1);
+
+        TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext);
+
+        verify(actorContext).setTxCreationLimit(Matchers.doubleThat(approximately(1000)));
+    }
+
     public Matcher<Double> approximately(final double val){
         return new BaseMatcher<Double>() {
             @Override
             public boolean matches(Object o) {
                 Double aDouble = (Double) o;
-                return aDouble > val && aDouble < val+1;
+                return aDouble >= val && aDouble <= val+1;
             }
 
             @Override

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.