From: Robert Varga Date: Fri, 10 Jun 2022 11:35:31 +0000 (+0200) Subject: Remove shard-snapshot-chunk-size X-Git-Tag: v9.0.0~15 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=3d2551546cd24632de113ad8c61d4d18b6e072ef Remove shard-snapshot-chunk-size Remove shard-snapshot-chunk-size and replace its occurrences with maximum-message-slice-size. JIRA: CONTROLLER-1980 Change-Id: Iab862d0789b75a89b6476b4ec048d7344362adf2 Signed-off-by: Robert Varga Signed-off-by: Matej Sramcik --- diff --git a/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/ExampleConfigParamsImpl.java b/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/ExampleConfigParamsImpl.java index 65d2109b30..3126156713 100644 --- a/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/ExampleConfigParamsImpl.java +++ b/opendaylight/md-sal/sal-akka-raft-example/src/main/java/org/opendaylight/controller/cluster/example/ExampleConfigParamsImpl.java @@ -19,7 +19,7 @@ public class ExampleConfigParamsImpl extends DefaultConfigParamsImpl { } @Override - public int getSnapshotChunkSize() { + public int getMaximumMessageSliceSize() { return 50; } } diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java index 8351374d60..7f98295a30 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/ConfigParams.java @@ -87,7 +87,7 @@ public interface ConfigParams { * * @return the maximum size (in bytes). */ - int getSnapshotChunkSize(); + int getMaximumMessageSliceSize(); /** * Returns the maximum number of journal log entries to batch on recovery before applying. diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java index 0d65fa2ea1..c83f90ec43 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/DefaultConfigParamsImpl.java @@ -41,7 +41,7 @@ public class DefaultConfigParamsImpl implements ConfigParams { */ private static final int ELECTION_TIME_MAX_VARIANCE = 100; - private static final int SNAPSHOT_CHUNK_SIZE = 480 * 1024; // 480KiB + private static final int MAXIMUM_MESSAGE_SLICE_SIZE = 480 * 1024; // 480KiB /** @@ -72,7 +72,7 @@ public class DefaultConfigParamsImpl implements ConfigParams { // 0 means direct threshold if disabled private int snapshotDataThreshold = 0; - private int snapshotChunkSize = SNAPSHOT_CHUNK_SIZE; + private int maximumMessageSliceSize = MAXIMUM_MESSAGE_SLICE_SIZE; private long electionTimeoutFactor = 2; private long candidateElectionTimeoutDivisor = 1; @@ -108,8 +108,8 @@ public class DefaultConfigParamsImpl implements ConfigParams { this.snapshotDataThreshold = snapshotDataThreshold; } - public void setSnapshotChunkSize(final int snapshotChunkSize) { - this.snapshotChunkSize = snapshotChunkSize; + public void setMaximumMessageSliceSize(final int maximumMessageSliceSize) { + this.maximumMessageSliceSize = maximumMessageSliceSize; } public void setJournalRecoveryLogBatchSize(final int journalRecoveryLogBatchSize) { @@ -191,8 +191,8 @@ public class DefaultConfigParamsImpl implements ConfigParams { } @Override - public int getSnapshotChunkSize() { - return snapshotChunkSize; + public int getMaximumMessageSliceSize() { + return maximumMessageSliceSize; } @Override diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java index e1a32ecef8..7514dccff4 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeader.java @@ -108,7 +108,7 @@ public abstract class AbstractLeader extends AbstractRaftActorBehavior { super(context, state); appendEntriesMessageSlicer = MessageSlicer.builder().logContext(logName()) - .messageSliceSize(context.getConfigParams().getSnapshotChunkSize()) + .messageSliceSize(context.getConfigParams().getMaximumMessageSliceSize()) .expireStateAfterInactivity(context.getConfigParams().getElectionTimeOutInterval().toMillis() * 3, TimeUnit.MILLISECONDS).build(); @@ -776,7 +776,7 @@ public abstract class AbstractLeader extends AbstractRaftActorBehavior { // Try to get all the entries in the journal but not exceeding the max data size for a single AppendEntries // message. int maxEntries = (int) context.getReplicatedLog().size(); - final int maxDataSize = context.getConfigParams().getSnapshotChunkSize(); + final int maxDataSize = context.getConfigParams().getMaximumMessageSliceSize(); final long followerNextIndex = followerLogInfo.getNextIndex(); List entries = context.getReplicatedLog().getFrom(followerNextIndex, maxEntries, maxDataSize); @@ -907,7 +907,7 @@ public abstract class AbstractLeader extends AbstractRaftActorBehavior { getReplicatedToAllIndex(), followerId); if (captureInitiated) { followerLogInfo.setLeaderInstallSnapshotState(new LeaderInstallSnapshotState( - context.getConfigParams().getSnapshotChunkSize(), logName())); + context.getConfigParams().getMaximumMessageSliceSize(), logName())); } return captureInitiated; @@ -949,8 +949,8 @@ public abstract class AbstractLeader extends AbstractRaftActorBehavior { if (snapshotHolder.isPresent()) { LeaderInstallSnapshotState installSnapshotState = followerLogInfo.getInstallSnapshotState(); if (installSnapshotState == null) { - installSnapshotState = new LeaderInstallSnapshotState(context.getConfigParams().getSnapshotChunkSize(), - logName()); + installSnapshotState = new LeaderInstallSnapshotState( + context.getConfigParams().getMaximumMessageSliceSize(), logName()); followerLogInfo.setLeaderInstallSnapshotState(installSnapshotState); } diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/AbstractRaftActorIntegrationTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/AbstractRaftActorIntegrationTest.java index 7b5ecc2d3b..f59598876d 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/AbstractRaftActorIntegrationTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/AbstractRaftActorIntegrationTest.java @@ -216,7 +216,7 @@ public abstract class AbstractRaftActorIntegrationTest extends AbstractActorTest } // FIXME: this is an arbitrary limit. Document interactions and/or improve them to improve maintainability - protected static final int SNAPSHOT_CHUNK_SIZE = 700; + protected static final int MAXIMUM_MESSAGE_SLICE_SIZE = 700; protected final Logger testLog = LoggerFactory.getLogger(getClass()); @@ -247,7 +247,7 @@ public abstract class AbstractRaftActorIntegrationTest extends AbstractActorTest protected long currentTerm; protected int snapshotBatchCount = 4; - protected int snapshotChunkSize = SNAPSHOT_CHUNK_SIZE; + protected int maximumMessageSliceSize = MAXIMUM_MESSAGE_SLICE_SIZE; protected List expSnapshotState = new ArrayList<>(); @@ -265,7 +265,7 @@ public abstract class AbstractRaftActorIntegrationTest extends AbstractActorTest configParams.setSnapshotBatchCount(snapshotBatchCount); configParams.setSnapshotDataThresholdPercentage(70); configParams.setIsolatedLeaderCheckInterval(new FiniteDuration(1, TimeUnit.DAYS)); - configParams.setSnapshotChunkSize(snapshotChunkSize); + configParams.setMaximumMessageSliceSize(maximumMessageSliceSize); return configParams; } diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest.java index ea4a5447e6..db4056ed4a 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest.java @@ -638,8 +638,8 @@ public class ReplicationAndSnapshotsWithLaggingFollowerIntegrationTest extends A assertEquals("Persisted Snapshot getUnAppliedEntries size", 0, unAppliedEntry.size()); int snapshotSize = SerializationUtils.serialize(persistedSnapshot.getState()).length; - final int expTotalChunks = snapshotSize / SNAPSHOT_CHUNK_SIZE - + (snapshotSize % SNAPSHOT_CHUNK_SIZE > 0 ? 1 : 0); + final int expTotalChunks = snapshotSize / MAXIMUM_MESSAGE_SLICE_SIZE + + (snapshotSize % MAXIMUM_MESSAGE_SLICE_SIZE > 0 ? 1 : 0); InstallSnapshot installSnapshot = MessageCollectorActor.expectFirstMatching(follower2CollectorActor, InstallSnapshot.class); diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationWithSlicedPayloadIntegrationTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationWithSlicedPayloadIntegrationTest.java index 10c11a7bc4..dcbc8179a6 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationWithSlicedPayloadIntegrationTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/ReplicationWithSlicedPayloadIntegrationTest.java @@ -29,7 +29,7 @@ public class ReplicationWithSlicedPayloadIntegrationTest extends AbstractRaftAct // Create the leader and 2 follower actors. - snapshotChunkSize = 20; + maximumMessageSliceSize = 20; DefaultConfigParamsImpl followerConfigParams = newFollowerConfigParams(); followerConfigParams.setSnapshotBatchCount(snapshotBatchCount); @@ -58,11 +58,11 @@ public class ReplicationWithSlicedPayloadIntegrationTest extends AbstractRaftAct // Send a large payload that exceeds the size threshold and needs to be sliced. - MockPayload largePayload = sendPayloadData(leaderActor, "large", snapshotChunkSize + 1); + MockPayload largePayload = sendPayloadData(leaderActor, "large", maximumMessageSliceSize + 1); // Then send a small payload that does not need to be sliced. - MockPayload smallPayload = sendPayloadData(leaderActor, "normal", snapshotChunkSize - 1); + MockPayload smallPayload = sendPayloadData(leaderActor, "normal", maximumMessageSliceSize - 1); final List leaderApplyState = expectMatching(leaderCollectorActor, ApplyState.class, 2); verifyApplyState(leaderApplyState.get(0), leaderCollectorActor, diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java index 4ca2abae0e..0f16f92c49 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java +++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/LeaderTest.java @@ -608,7 +608,7 @@ public class LeaderTest extends AbstractLeaderTest { List.of(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null), ByteSource.wrap(bs.toByteArray()))); LeaderInstallSnapshotState fts = new LeaderInstallSnapshotState( - actorContext.getConfigParams().getSnapshotChunkSize(), leader.logName()); + actorContext.getConfigParams().getMaximumMessageSliceSize(), leader.logName()); fts.setSnapshotBytes(ByteSource.wrap(bs.toByteArray())); leader.getFollower(FOLLOWER_ID).setLeaderInstallSnapshotState(fts); @@ -965,7 +965,7 @@ public class LeaderTest extends AbstractLeaderTest { List.of(), commitIndex, snapshotTerm, commitIndex, snapshotTerm, -1, null, null), ByteSource.wrap(bs.toByteArray()))); LeaderInstallSnapshotState fts = new LeaderInstallSnapshotState( - actorContext.getConfigParams().getSnapshotChunkSize(), leader.logName()); + actorContext.getConfigParams().getMaximumMessageSliceSize(), leader.logName()); fts.setSnapshotBytes(ByteSource.wrap(bs.toByteArray())); leader.getFollower(FOLLOWER_ID).setLeaderInstallSnapshotState(fts); while (!fts.isLastChunk(fts.getChunkIndex())) { @@ -1003,7 +1003,7 @@ public class LeaderTest extends AbstractLeaderTest { DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl() { @Override - public int getSnapshotChunkSize() { + public int getMaximumMessageSliceSize() { return 50; } }; @@ -1080,7 +1080,7 @@ public class LeaderTest extends AbstractLeaderTest { actorContext.setConfigParams(new DefaultConfigParamsImpl() { @Override - public int getSnapshotChunkSize() { + public int getMaximumMessageSliceSize() { return 50; } }); @@ -1144,7 +1144,7 @@ public class LeaderTest extends AbstractLeaderTest { actorContext.setConfigParams(new DefaultConfigParamsImpl() { @Override - public int getSnapshotChunkSize() { + public int getMaximumMessageSliceSize() { return 50; } }); @@ -1772,7 +1772,7 @@ public class LeaderTest extends AbstractLeaderTest { ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setHeartBeatInterval( new FiniteDuration(1000, TimeUnit.SECONDS)); // Note: the size here depends on estimate - ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setSnapshotChunkSize(246); + ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setMaximumMessageSliceSize(246); leaderActorContext.setReplicatedLog( new MockRaftActorContext.MockReplicatedLogBuilder().createEntries(0, 4, 1).build()); @@ -2255,7 +2255,7 @@ public class LeaderTest extends AbstractLeaderTest { MockRaftActorContext leaderActorContext = createActorContextWithFollower(); ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setHeartBeatInterval( new FiniteDuration(300, TimeUnit.MILLISECONDS)); - ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setSnapshotChunkSize(serializedSize - 50); + ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setMaximumMessageSliceSize(serializedSize - 50); leaderActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build()); leaderActorContext.setCommitIndex(-1); leaderActorContext.setLastApplied(-1); @@ -2339,7 +2339,7 @@ public class LeaderTest extends AbstractLeaderTest { ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setHeartBeatInterval( new FiniteDuration(100, TimeUnit.MILLISECONDS)); ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setElectionTimeoutFactor(1); - ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setSnapshotChunkSize(10); + ((DefaultConfigParamsImpl)leaderActorContext.getConfigParams()).setMaximumMessageSliceSize(10); leaderActorContext.setReplicatedLog(new MockRaftActorContext.MockReplicatedLogBuilder().build()); leaderActorContext.setCommitIndex(-1); leaderActorContext.setLastApplied(-1); @@ -2354,7 +2354,7 @@ public class LeaderTest extends AbstractLeaderTest { MessageCollectorActor.clearMessages(followerActor); sendReplicate(leaderActorContext, term, 0, new MockRaftActorContext.MockPayload("large", - leaderActorContext.getConfigParams().getSnapshotChunkSize() + 1)); + leaderActorContext.getConfigParams().getMaximumMessageSliceSize() + 1)); MessageCollectorActor.expectFirstMatching(followerActor, MessageSlice.class); // Sleep for at least 3 * election timeout so the slicing state expires. @@ -2443,11 +2443,11 @@ public class LeaderTest extends AbstractLeaderTest { private static class MockConfigParamsImpl extends DefaultConfigParamsImpl { private final long electionTimeOutIntervalMillis; - private final int snapshotChunkSize; + private final int maximumMessageSliceSize; - MockConfigParamsImpl(final long electionTimeOutIntervalMillis, final int snapshotChunkSize) { + MockConfigParamsImpl(final long electionTimeOutIntervalMillis, final int maximumMessageSliceSize) { this.electionTimeOutIntervalMillis = electionTimeOutIntervalMillis; - this.snapshotChunkSize = snapshotChunkSize; + this.maximumMessageSliceSize = maximumMessageSliceSize; } @Override @@ -2456,8 +2456,8 @@ public class LeaderTest extends AbstractLeaderTest { } @Override - public int getSnapshotChunkSize() { - return snapshotChunkSize; + public int getMaximumMessageSliceSize() { + return maximumMessageSliceSize; } } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java index 4b55dda597..24b3775127 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java @@ -165,7 +165,6 @@ public class DatastoreContext implements ClientActorConfig { setCandidateElectionTimeoutDivisor(other.raftConfig.getCandidateElectionTimeoutDivisor()); setCustomRaftPolicyImplementation(other.raftConfig.getCustomRaftPolicyImplementationClass()); setMaximumMessageSliceSize(other.getMaximumMessageSliceSize()); - setShardSnapshotChunkSize(other.raftConfig.getSnapshotChunkSize()); setPeerAddressResolver(other.raftConfig.getPeerAddressResolver()); setTempFileDirectory(other.getTempFileDirectory()); setFileBackedStreamingThreshold(other.getFileBackedStreamingThreshold()); @@ -329,17 +328,8 @@ public class DatastoreContext implements ClientActorConfig { raftConfig.setRecoverySnapshotIntervalSeconds(recoverySnapshotInterval); } - @Deprecated - private void setShardSnapshotChunkSize(final int shardSnapshotChunkSize) { - // We'll honor the shardSnapshotChunkSize setting for backwards compatibility but only if it doesn't exceed - // maximumMessageSliceSize. - if (shardSnapshotChunkSize < maximumMessageSliceSize) { - raftConfig.setSnapshotChunkSize(shardSnapshotChunkSize); - } - } - private void setMaximumMessageSliceSize(final int maximumMessageSliceSize) { - raftConfig.setSnapshotChunkSize(maximumMessageSliceSize); + raftConfig.setMaximumMessageSliceSize(maximumMessageSliceSize); this.maximumMessageSliceSize = maximumMessageSliceSize; } @@ -625,14 +615,6 @@ public class DatastoreContext implements ClientActorConfig { return this; } - @Deprecated - public Builder shardSnapshotChunkSize(final int shardSnapshotChunkSize) { - LOG.warn("The shard-snapshot-chunk-size configuration parameter is deprecated - " - + "use maximum-message-slice-size instead"); - datastoreContext.setShardSnapshotChunkSize(shardSnapshotChunkSize); - return this; - } - public Builder maximumMessageSliceSize(final int maximumMessageSliceSize) { datastoreContext.setMaximumMessageSliceSize(maximumMessageSliceSize); return this; diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang b/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang index b8c570de79..21307e532c 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/yang/distributed-datastore-provider.yang @@ -9,7 +9,7 @@ module distributed-datastore-provider { the distributed datastore provider implementation"; revision "2023-12-29" { - description "Remote use-tell-based-protocol leaf"; + description "Remote use-tell-based-protocol and shard-snapshot-chunk-size leaves"; } revision "2014-06-12" { @@ -212,14 +212,6 @@ module distributed-datastore-provider { cannot be found then the default raft behavior will be applied"; } - leaf shard-snapshot-chunk-size { - status deprecated; - default 491520; - type non-zero-uint32-type; - description "When sending a snapshot to a follower, this is the maximum size in bytes for - a chunk of data."; - } - leaf maximum-message-slice-size { default 491520; type non-zero-uint32-type;