Add direct in-memory journal threshold 99/91299/10
authortadei.bilan <tadei.bilan@pantheon.tech>
Tue, 21 Jul 2020 09:24:09 +0000 (12:24 +0300)
committerRobert Varga <robert.varga@pantheon.tech>
Thu, 3 Sep 2020 10:14:01 +0000 (12:14 +0200)
Some deployments benefit from placing an absolute numeric limit
on the retained memory. Introduce a new tunable, which overrides
the usual percentange limit.

JIRA: CONTROLLER-1956
Change-Id: I688e226b173386765bea74931b6aaf617bda30a8
Signed-off-by: tadei.bilan <tadei.bilan@pantheon.tech>
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ReplicatedLogImpl.java
opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/SnapshotManager.java
opendaylight/md-sal/sal-clustering-config/src/main/resources/initial/datastore.cfg
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreConfigurationMXBean.java
opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/jmx/mbeans/DatastoreConfigurationMXBeanImpl.java
opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DatastoreContextIntrospectorTest.java
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DatastoreContextTest.java

index c5c7813..8351374 100644 (file)
@@ -12,14 +12,9 @@ import org.opendaylight.controller.cluster.raft.policy.RaftPolicy;
 import scala.concurrent.duration.FiniteDuration;
 
 /**
- * Configuration Parameter interface for configuring the Raft consensus system
- *
- * <p>
- * Any component using this implementation might want to provide an implementation of
- * this interface to configure
- *
- * <p>
- * A default implementation will be used if none is provided.
+ * Configuration Parameter interface for configuring the Raft consensus system. Any component using this implementation
+ * might want to provide an implementation of this interface to configure. A default implementation will be used if none
+ * is provided.
  *
  * @author Kamal Rameshan
  */
@@ -35,15 +30,23 @@ public interface ConfigParams {
 
     /**
      * Returns the percentage of total memory used in the in-memory Raft log before a snapshot should be taken.
+     * Disabled when direct threshold is enabled.
      *
      * @return the percentage.
      */
     int getSnapshotDataThresholdPercentage();
 
+    /**
+     * Returns the max size of memory used in the in-memory Raft log before a snapshot should be taken. 0 means that
+     * direct threshold is disabled and percentage is used instead.
+     *
+     * @return maximum journal size (in MiB).
+     */
+    int getSnapshotDataThreshold();
 
     /**
-     * Returns the interval(in seconds) after which a snapshot should be taken during recovery.
-     * Negative value means don't take snapshots.
+     * Returns the interval(in seconds) after which a snapshot should be taken during recovery. Negative value means
+     * do not take snapshots.
      *
      * @return the interval of recovery snapshot in seconds
      */
@@ -100,7 +103,6 @@ public interface ConfigParams {
      */
     long getIsolatedCheckIntervalInMillis();
 
-
     /**
      * Returns the multiplication factor to be used to determine the shard election timeout. The election timeout
      * is determined by multiplying the election timeout factor with the heart beat duration.
@@ -109,7 +111,6 @@ public interface ConfigParams {
      */
     long getElectionTimeoutFactor();
 
-
     /**
      * Returns the RaftPolicy used to determine certain Raft behaviors.
      *
index 97838b0..37ed729 100644 (file)
@@ -68,6 +68,10 @@ public class DefaultConfigParamsImpl implements ConfigParams {
     // in-memory journal can use before it needs to snapshot
     private int snapshotDataThresholdPercentage = 12;
 
+    // max size of in-memory journal in MB
+    // 0 means direct threshold if disabled
+    private int snapshotDataThreshold = 0;
+
     private int snapshotChunkSize = SNAPSHOT_CHUNK_SIZE;
 
     private long electionTimeoutFactor = 2;
@@ -100,6 +104,10 @@ public class DefaultConfigParamsImpl implements ConfigParams {
         this.snapshotDataThresholdPercentage = snapshotDataThresholdPercentage;
     }
 
+    public void setSnapshotDataThreshold(final int snapshotDataThreshold) {
+        this.snapshotDataThreshold = snapshotDataThreshold;
+    }
+
     public void setSnapshotChunkSize(final int snapshotChunkSize) {
         this.snapshotChunkSize = snapshotChunkSize;
     }
@@ -148,6 +156,11 @@ public class DefaultConfigParamsImpl implements ConfigParams {
         return snapshotDataThresholdPercentage;
     }
 
+    @Override
+    public int getSnapshotDataThreshold() {
+        return snapshotDataThreshold;
+    }
+
     @Override
     public int getRecoverySnapshotIntervalSeconds() {
         return this.recoverySnapshotIntervalSeconds;
index 66d7b35..6167aac 100644 (file)
@@ -54,10 +54,14 @@ final class ReplicatedLogImpl extends AbstractReplicatedLogImpl {
     @Override
     public boolean shouldCaptureSnapshot(final long logIndex) {
         final ConfigParams config = context.getConfigParams();
-        final long journalSize = logIndex + 1;
-        final long dataThreshold = context.getTotalMemory() * config.getSnapshotDataThresholdPercentage() / 100;
+        if ((logIndex + 1) % config.getSnapshotBatchCount() == 0) {
+            return true;
+        }
 
-        return journalSize % config.getSnapshotBatchCount() == 0 || getDataSizeForSnapshotCheck() > dataThreshold;
+        final long absoluteThreshold = config.getSnapshotDataThreshold();
+        final long dataThreshold = absoluteThreshold != 0 ? absoluteThreshold * ConfigParams.MEGABYTE
+                : context.getTotalMemory() * config.getSnapshotDataThresholdPercentage() / 100;
+        return getDataSizeForSnapshotCheck() > dataThreshold;
     }
 
     @Override
index 3e2410e..791a791 100644 (file)
@@ -379,11 +379,14 @@ public class SnapshotManager implements SnapshotState {
 
             log.info("{}: Persisting of snapshot done: {}", persistenceId(), snapshot);
 
-            long dataThreshold = totalMemory * context.getConfigParams().getSnapshotDataThresholdPercentage() / 100;
-            boolean dataSizeThresholdExceeded = context.getReplicatedLog().dataSize() > dataThreshold;
+            final ConfigParams config = context.getConfigParams();
+            final long absoluteThreshold = config.getSnapshotDataThreshold();
+            final long dataThreshold = absoluteThreshold != 0 ? absoluteThreshold * ConfigParams.MEGABYTE
+                    : totalMemory * config.getSnapshotDataThresholdPercentage() / 100;
 
-            boolean logSizeExceededSnapshotBatchCount =
-                    context.getReplicatedLog().size() >= context.getConfigParams().getSnapshotBatchCount();
+            final boolean dataSizeThresholdExceeded = context.getReplicatedLog().dataSize() > dataThreshold;
+            final boolean logSizeExceededSnapshotBatchCount =
+                    context.getReplicatedLog().size() >= config.getSnapshotBatchCount();
 
             final RaftActorBehavior currentBehavior = context.getCurrentBehavior();
             if (dataSizeThresholdExceeded || logSizeExceededSnapshotBatchCount || captureSnapshot.isMandatoryTrim()) {
@@ -395,8 +398,7 @@ public class SnapshotManager implements SnapshotState {
                     } else if (logSizeExceededSnapshotBatchCount) {
                         log.debug("{}: log size {} exceeds the snapshot batch count {} - doing snapshotPreCommit with "
                                 + "index {}", context.getId(), context.getReplicatedLog().size(),
-                                context.getConfigParams().getSnapshotBatchCount(),
-                                captureSnapshot.getLastAppliedIndex());
+                                config.getSnapshotBatchCount(), captureSnapshot.getLastAppliedIndex());
                     } else {
                         log.debug("{}: user triggered or root overwrite snapshot encountered, trimming log up to "
                                 + "last applied index {}", context.getId(), captureSnapshot.getLastAppliedIndex());
index 7d4903e..e65c8f7 100644 (file)
@@ -53,8 +53,13 @@ operational.persistent=false
 #shard-snapshot-batch-count=20000
 
 # The percentage of Runtime.totalMemory() used by the in-memory journal log before a snapshot is to be taken.
+# Disabled, if direct threshold is enabled.
 #shard-snapshot-data-threshold-percentage=12
 
+# The max size of in-memory journal(in MB), after reaching the limit, snapshot will be taken. Should be not less then 1.
+# If set to 0, direct threshold is disabled and percentage is used instead.
+#shard-snapshot-data-threshold=0
+
 # The interval at which the leader of the shard will check if its majority followers are active and
 # term itself as isolated.
 #shard-isolated-leader-check-interval-in-millis=5000
index 69131a0..2987ea2 100644 (file)
@@ -56,6 +56,7 @@ public class DatastoreContext implements ClientActorConfig {
     public static final boolean DEFAULT_SNAPSHOT_ON_ROOT_OVERWRITE = false;
     public static final FileAkkaConfigurationReader DEFAULT_CONFIGURATION_READER = new FileAkkaConfigurationReader();
     public static final int DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE = 12;
+    public static final int DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD = 0;
     public static final int DEFAULT_SHARD_ELECTION_TIMEOUT_FACTOR = 2;
     public static final int DEFAULT_SHARD_CANDIDATE_ELECTION_TIMEOUT_DIVISOR = 1;
     public static final int DEFAULT_TX_CREATION_INITIAL_RATE_LIMIT = 100;
@@ -113,6 +114,7 @@ public class DatastoreContext implements ClientActorConfig {
         setHeartbeatInterval(DEFAULT_HEARTBEAT_INTERVAL_IN_MILLIS);
         setIsolatedLeaderCheckInterval(DEFAULT_ISOLATED_LEADER_CHECK_INTERVAL_IN_MILLIS);
         setSnapshotDataThresholdPercentage(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE);
+        setSnapshotDataThreshold(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD);
         setElectionTimeoutFactor(DEFAULT_SHARD_ELECTION_TIMEOUT_FACTOR);
         setCandidateElectionTimeoutDivisor(DEFAULT_SHARD_CANDIDATE_ELECTION_TIMEOUT_DIVISOR);
         setSyncIndexThreshold(DEFAULT_SYNC_INDEX_THRESHOLD);
@@ -153,6 +155,7 @@ public class DatastoreContext implements ClientActorConfig {
         setHeartbeatInterval(other.raftConfig.getHeartBeatInterval().toMillis());
         setIsolatedLeaderCheckInterval(other.raftConfig.getIsolatedCheckIntervalInMillis());
         setSnapshotDataThresholdPercentage(other.raftConfig.getSnapshotDataThresholdPercentage());
+        setSnapshotDataThreshold(other.raftConfig.getSnapshotDataThreshold());
         setElectionTimeoutFactor(other.raftConfig.getElectionTimeoutFactor());
         setCandidateElectionTimeoutDivisor(other.raftConfig.getCandidateElectionTimeoutDivisor());
         setCustomRaftPolicyImplementation(other.raftConfig.getCustomRaftPolicyImplementationClass());
@@ -304,6 +307,11 @@ public class DatastoreContext implements ClientActorConfig {
         raftConfig.setSnapshotDataThresholdPercentage(shardSnapshotDataThresholdPercentage);
     }
 
+    private void setSnapshotDataThreshold(final int shardSnapshotDataThreshold) {
+        checkArgument(shardSnapshotDataThreshold >= 0);
+        raftConfig.setSnapshotDataThreshold(shardSnapshotDataThreshold);
+    }
+
     private void setSnapshotBatchCount(final long shardSnapshotBatchCount) {
         raftConfig.setSnapshotBatchCount(shardSnapshotBatchCount);
     }
@@ -450,6 +458,11 @@ public class DatastoreContext implements ClientActorConfig {
             return this;
         }
 
+        public Builder shardSnapshotDataThreshold(final int shardSnapshotDataThreshold) {
+            datastoreContext.setSnapshotDataThreshold(shardSnapshotDataThreshold);
+            return this;
+        }
+
         public Builder shardHeartbeatIntervalInMillis(final int shardHeartbeatIntervalInMillis) {
             datastoreContext.setHeartbeatInterval(shardHeartbeatIntervalInMillis);
             return this;
index e17f403..1bef70a 100644 (file)
@@ -64,6 +64,11 @@ public class DatastoreConfigurationMXBeanImpl extends AbstractMXBean implements
         return context.getShardRaftConfig().getSnapshotDataThresholdPercentage();
     }
 
+    @Override
+    public int getShardSnapshotDataThreshold() {
+        return context.getShardRaftConfig().getSnapshotDataThreshold();
+    }
+
     @Override
     public long getShardSnapshotBatchCount() {
         return context.getShardRaftConfig().getSnapshotBatchCount();
index f0d6693..da0c748 100644 (file)
@@ -78,7 +78,17 @@ module distributed-datastore-provider {
         leaf shard-snapshot-data-threshold-percentage {
             default 12;
             type percentage;
-            description "The percentage of Runtime.maxMemory() used by the in-memory journal log before a snapshot is to be taken";
+            description "The percentage of Runtime.maxMemory() used by the in-memory journal log before a snapshot is to be taken.
+                         Disabled, if direct threshold is enabled.";
+        }
+
+        leaf shard-snapshot-data-threshold {
+            default 0;
+            type uint32 {
+                range "0..max";
+            }
+            description "The threshold of in-memory journal size before a snapshot is to be taken. If set to 0, direct threshold
+                         is disabled and percentage is used instead.";
         }
 
 
index ba62f8d..aab9322 100644 (file)
@@ -13,6 +13,7 @@ import static org.junit.Assert.assertTrue;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_HEARTBEAT_INTERVAL_IN_MILLIS;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_OPERATION_TIMEOUT_IN_MS;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_INITIALIZATION_TIMEOUT;
+import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_TRANSACTION_IDLE_TIMEOUT;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_TX_COMMIT_TIMEOUT_IN_SECONDS;
@@ -65,6 +66,7 @@ public class DatastoreContextIntrospectorTest {
         properties.put("recovery-snapshot-interval-seconds", "360");
         properties.put("shard-isolated-leader-check-interval-in-millis", "123");
         properties.put("shard-snapshot-data-threshold-percentage", "100");
+        properties.put("shard-snapshot-data-threshold", "800");
         properties.put("shard-election-timeout-factor", "21");
         properties.put("shard-batched-modification-count", "901");
         properties.put("transactionCreationInitialRateLimit", "200");
@@ -92,6 +94,7 @@ public class DatastoreContextIntrospectorTest {
         assertEquals(360, context.getShardRaftConfig().getRecoverySnapshotIntervalSeconds());
         assertEquals(123, context.getShardRaftConfig().getIsolatedCheckIntervalInMillis());
         assertEquals(100, context.getShardRaftConfig().getSnapshotDataThresholdPercentage());
+        assertEquals(800, context.getShardRaftConfig().getSnapshotDataThreshold());
         assertEquals(21, context.getShardRaftConfig().getElectionTimeoutFactor());
         assertEquals(901, context.getShardBatchedModificationCount());
         assertEquals(200, context.getTransactionCreationInitialRateLimit());
@@ -123,6 +126,7 @@ public class DatastoreContextIntrospectorTest {
         assertEquals(6, context.getInitialSettleTimeoutMultiplier());
         assertEquals(123, context.getShardRaftConfig().getIsolatedCheckIntervalInMillis());
         assertEquals(100, context.getShardRaftConfig().getSnapshotDataThresholdPercentage());
+        assertEquals(800, context.getShardRaftConfig().getSnapshotDataThreshold());
         assertEquals(22, context.getShardRaftConfig().getElectionTimeoutFactor());
         assertEquals(200, context.getTransactionCreationInitialRateLimit());
         assertTrue(context.isPersistent());
@@ -148,6 +152,7 @@ public class DatastoreContextIntrospectorTest {
         properties.put("shard-heartbeat-interval-in-millis", "99"); // bad - must be >= 100
         properties.put("shard-transaction-commit-queue-capacity", "567"); // good
         properties.put("shard-snapshot-data-threshold-percentage", "101"); // bad - must be 0-100
+        properties.put("shard-snapshot-data-threshold", "-1"); // bad - must be > 0
         properties.put("shard-initialization-timeout-in-seconds", "-1"); // bad - must be > 0
         properties.put("max-shard-data-change-executor-pool-size", "bogus"); // bad - NaN
         properties.put("unknownProperty", "1"); // bad - invalid property name
@@ -166,6 +171,7 @@ public class DatastoreContextIntrospectorTest {
         assertEquals(567, context.getShardTransactionCommitQueueCapacity());
         assertEquals(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE,
                 context.getShardRaftConfig().getSnapshotDataThresholdPercentage());
+        assertEquals(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD, context.getShardRaftConfig().getSnapshotDataThreshold());
         assertEquals(DEFAULT_SHARD_INITIALIZATION_TIMEOUT, context.getShardInitializationTimeout());
     }
 
index 6cab9db..4d47f7f 100644 (file)
@@ -22,6 +22,7 @@ import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEF
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_ELECTION_TIMEOUT_FACTOR;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_INITIALIZATION_TIMEOUT;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_LEADER_ELECTION_TIMEOUT;
+import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_TRANSACTION_IDLE_TIMEOUT;
 import static org.opendaylight.controller.cluster.datastore.DatastoreContext.DEFAULT_SHARD_TX_COMMIT_QUEUE_CAPACITY;
@@ -61,6 +62,8 @@ public class DatastoreContextTest {
                 context.getShardRaftConfig().getIsolatedCheckIntervalInMillis());
         assertEquals(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE,
                 context.getShardRaftConfig().getSnapshotDataThresholdPercentage());
+        assertEquals(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD,
+                context.getShardRaftConfig().getSnapshotDataThreshold());
         assertEquals(DEFAULT_SHARD_ELECTION_TIMEOUT_FACTOR, context.getShardRaftConfig().getElectionTimeoutFactor());
         assertEquals(DEFAULT_TX_CREATION_INITIAL_RATE_LIMIT, context.getTransactionCreationInitialRateLimit());
         assertEquals(DatastoreContext.DEFAULT_SHARD_BATCHED_MODIFICATION_COUNT,
@@ -91,6 +94,7 @@ public class DatastoreContextTest {
         builder.persistent(!DEFAULT_PERSISTENT);
         builder.shardIsolatedLeaderCheckIntervalInMillis(DEFAULT_ISOLATED_LEADER_CHECK_INTERVAL_IN_MILLIS + 1);
         builder.shardSnapshotDataThresholdPercentage(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE + 1);
+        builder.shardSnapshotDataThreshold(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD + 1);
         builder.shardElectionTimeoutFactor(DEFAULT_SHARD_ELECTION_TIMEOUT_FACTOR + 1);
         builder.transactionCreationInitialRateLimit(DEFAULT_TX_CREATION_INITIAL_RATE_LIMIT + 1);
         builder.shardBatchedModificationCount(DEFAULT_SHARD_BATCHED_MODIFICATION_COUNT + 1);
@@ -145,6 +149,8 @@ public class DatastoreContextTest {
                 context.getShardRaftConfig().getIsolatedCheckIntervalInMillis());
         assertEquals(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD_PERCENTAGE + 1,
                 context.getShardRaftConfig().getSnapshotDataThresholdPercentage());
+        assertEquals(DEFAULT_SHARD_SNAPSHOT_DATA_THRESHOLD + 1,
+                context.getShardRaftConfig().getSnapshotDataThreshold());
         assertEquals(DEFAULT_SHARD_ELECTION_TIMEOUT_FACTOR + 1,
                 context.getShardRaftConfig().getElectionTimeoutFactor());
         assertEquals(DEFAULT_TX_CREATION_INITIAL_RATE_LIMIT + 1, context.getTransactionCreationInitialRateLimit());

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.