From 9905bf0575ff196a531eb114e89b1bdb7226bc6c Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Tue, 4 Dec 2018 13:02:38 -0500 Subject: [PATCH] Specify initial serialization buffer capacity for Payloads JFR shows a lot of re-allocations of the backing byte [] when serializing CommitTransactionPayload. Specify a reasonable initial buffer capacity (the default in ByteArrayOutputStream is 32) to reduce re-allocations. This is also configurable via the cfg file (default is 512). Also did the same for the other smaller Payload classes like PurgeTransactionPayload. JIRA: CONTROLLER-1870 Change-Id: I7ebced56812bfc102409b5b2a8b7f4b1b54359fc Signed-off-by: Tom Pantelis --- .../cluster/datastore/DatastoreContext.java | 14 +++++++++++- .../controller/cluster/datastore/Shard.java | 3 ++- .../cluster/datastore/ShardDataTree.java | 18 ++++++++++----- .../persisted/AbortTransactionPayload.java | 5 +++-- .../persisted/CloseLocalHistoryPayload.java | 5 +++-- .../persisted/CommitTransactionPayload.java | 22 ++++++++++++++++--- .../persisted/CreateLocalHistoryPayload.java | 5 +++-- .../persisted/PurgeLocalHistoryPayload.java | 5 +++-- .../persisted/PurgeTransactionPayload.java | 5 +++-- .../yang/distributed-datastore-provider.yang | 6 +++++ .../DatastoreContextIntrospectorTest.java | 2 ++ .../datastore/DatastoreContextTest.java | 4 ++++ .../cluster/datastore/ShardDataTreeTest.java | 10 +++++---- .../AbortTransactionPayloadTest.java | 2 +- .../CloseLocalHistoryPayloadTest.java | 4 ++-- .../CreateLocalHistoryPayloadTest.java | 4 ++-- .../PurgeLocalHistoryPayloadTest.java | 4 ++-- .../PurgeTransactionPayloadTest.java | 4 ++-- 18 files changed, 88 insertions(+), 34 deletions(-) diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java index e67ef53296..98afd7f4e0 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DatastoreContext.java @@ -33,7 +33,7 @@ import scala.concurrent.duration.FiniteDuration; * * @author Thomas Pantelis */ -// Noo-final for mocking +// Non-final for mocking public class DatastoreContext implements ClientActorConfig { public static final String METRICS_DOMAIN = "org.opendaylight.controller.cluster.datastore"; @@ -59,6 +59,7 @@ public class DatastoreContext implements ClientActorConfig { public static final long DEFAULT_SHARD_COMMIT_QUEUE_EXPIRY_TIMEOUT_IN_MS = TimeUnit.MILLISECONDS.convert(2, TimeUnit.MINUTES); public static final int DEFAULT_MAX_MESSAGE_SLICE_SIZE = 2048 * 1000; // 2MB + public static final int DEFAULT_INITIAL_PAYLOAD_SERIALIZED_BUFFER_CAPACITY = 512; public static final long DEFAULT_SYNC_INDEX_THRESHOLD = 10; @@ -92,6 +93,7 @@ public class DatastoreContext implements ClientActorConfig { private long backendAlivenessTimerInterval = AbstractClientConnection.DEFAULT_BACKEND_ALIVE_TIMEOUT_NANOS; private long requestTimeout = AbstractClientConnection.DEFAULT_REQUEST_TIMEOUT_NANOS; private long noProgressTimeout = AbstractClientConnection.DEFAULT_NO_PROGRESS_TIMEOUT_NANOS; + private int initialPayloadSerializedBufferCapacity = DEFAULT_INITIAL_PAYLOAD_SERIALIZED_BUFFER_CAPACITY; public static Set getGlobalDatastoreNames() { return GLOBAL_DATASTORE_NAMES; @@ -132,6 +134,7 @@ public class DatastoreContext implements ClientActorConfig { this.backendAlivenessTimerInterval = other.backendAlivenessTimerInterval; this.requestTimeout = other.requestTimeout; this.noProgressTimeout = other.noProgressTimeout; + this.initialPayloadSerializedBufferCapacity = other.initialPayloadSerializedBufferCapacity; setShardJournalRecoveryLogBatchSize(other.raftConfig.getJournalRecoveryLogBatchSize()); setSnapshotBatchCount(other.raftConfig.getSnapshotBatchCount()); @@ -337,6 +340,10 @@ public class DatastoreContext implements ClientActorConfig { return noProgressTimeout; } + public int getInitialPayloadSerializedBufferCapacity() { + return initialPayloadSerializedBufferCapacity; + } + public static class Builder implements org.opendaylight.yangtools.concepts.Builder { private final DatastoreContext datastoreContext; private int maxShardDataChangeExecutorPoolSize = @@ -613,6 +620,11 @@ public class DatastoreContext implements ClientActorConfig { return this; } + public Builder initialPayloadSerializedBufferCapacity(final int capacity) { + datastoreContext.initialPayloadSerializedBufferCapacity = capacity; + return this; + } + @Override public DatastoreContext build() { datastoreContext.dataStoreProperties = InMemoryDOMDataStoreConfigProperties.builder() diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java index 334bd8e2d2..51ee4d7b80 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java @@ -363,7 +363,8 @@ public class Shard extends RaftActor { (DataTreeCohortActorRegistry.CohortRegistryCommand) message); } else if (message instanceof PersistAbortTransactionPayload) { final TransactionIdentifier txId = ((PersistAbortTransactionPayload) message).getTransactionId(); - persistPayload(txId, AbortTransactionPayload.create(txId), true); + persistPayload(txId, AbortTransactionPayload.create( + txId, datastoreContext.getInitialPayloadSerializedBufferCapacity()), true); } else if (message instanceof MakeLeaderLocal) { onMakeLeaderLocal(); } else if (RESUME_NEXT_PENDING_TRANSACTION.equals(message)) { diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java index 92263f2102..8b826015c4 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java @@ -537,7 +537,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent { if (chain == null) { chain = new ShardDataTreeTransactionChain(historyId, this); transactionChains.put(historyId, chain); - replicatePayload(historyId, CreateLocalHistoryPayload.create(historyId), callback); + replicatePayload(historyId, CreateLocalHistoryPayload.create( + historyId, shard.getDatastoreContext().getInitialPayloadSerializedBufferCapacity()), callback); } else if (callback != null) { callback.run(); } @@ -597,7 +598,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent { } chain.close(); - replicatePayload(id, CloseLocalHistoryPayload.create(id), callback); + replicatePayload(id, CloseLocalHistoryPayload.create( + id, shard.getDatastoreContext().getInitialPayloadSerializedBufferCapacity()), callback); } /** @@ -616,7 +618,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent { return; } - replicatePayload(id, PurgeLocalHistoryPayload.create(id), callback); + replicatePayload(id, PurgeLocalHistoryPayload.create( + id, shard.getDatastoreContext().getInitialPayloadSerializedBufferCapacity()), callback); } Optional readCurrentData() { @@ -640,7 +643,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent { void abortTransaction(final AbstractShardDataTreeTransaction transaction, final Runnable callback) { final TransactionIdentifier id = transaction.getIdentifier(); LOG.debug("{}: aborting transaction {}", logContext, id); - replicatePayload(id, AbortTransactionPayload.create(id), callback); + replicatePayload(id, AbortTransactionPayload.create( + id, shard.getDatastoreContext().getInitialPayloadSerializedBufferCapacity()), callback); } @Override @@ -660,7 +664,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent { void purgeTransaction(final TransactionIdentifier id, final Runnable callback) { LOG.debug("{}: purging transaction {}", logContext, id); - replicatePayload(id, PurgeTransactionPayload.create(id), callback); + replicatePayload(id, PurgeTransactionPayload.create( + id, shard.getDatastoreContext().getInitialPayloadSerializedBufferCapacity()), callback); } public Optional> readNode(final YangInstanceIdentifier path) { @@ -999,7 +1004,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent { final TransactionIdentifier txId = cohort.getIdentifier(); final Payload payload; try { - payload = CommitTransactionPayload.create(txId, candidate); + payload = CommitTransactionPayload.create(txId, candidate, + shard.getDatastoreContext().getInitialPayloadSerializedBufferCapacity()); } catch (IOException e) { LOG.error("{}: Failed to encode transaction {} candidate {}", logContext, txId, candidate, e); pendingCommits.poll().cohort.failedCommit(e); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/AbortTransactionPayload.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/AbortTransactionPayload.java index ad9bc4a7e8..0e34756ced 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/AbortTransactionPayload.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/AbortTransactionPayload.java @@ -54,8 +54,9 @@ public final class AbortTransactionPayload extends AbstractIdentifiablePayload getCandidate() throws IOException { diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/CreateLocalHistoryPayload.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/CreateLocalHistoryPayload.java index b7f90e32d7..dbf72f38d8 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/CreateLocalHistoryPayload.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/CreateLocalHistoryPayload.java @@ -54,8 +54,9 @@ public final class CreateLocalHistoryPayload extends AbstractIdentifiablePayload super(historyId, serialized); } - public static CreateLocalHistoryPayload create(final LocalHistoryIdentifier historyId) { - final ByteArrayDataOutput out = ByteStreams.newDataOutput(); + public static CreateLocalHistoryPayload create(final LocalHistoryIdentifier historyId, + final int initialSerializedBufferCapacity) { + final ByteArrayDataOutput out = ByteStreams.newDataOutput(initialSerializedBufferCapacity); try { historyId.writeTo(out); } catch (IOException e) { diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeLocalHistoryPayload.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeLocalHistoryPayload.java index c555f3ca6d..8d9a8d217a 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeLocalHistoryPayload.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeLocalHistoryPayload.java @@ -55,8 +55,9 @@ public final class PurgeLocalHistoryPayload extends AbstractIdentifiablePayload< super(historyId, serialized); } - public static PurgeLocalHistoryPayload create(final LocalHistoryIdentifier historyId) { - final ByteArrayDataOutput out = ByteStreams.newDataOutput(); + public static PurgeLocalHistoryPayload create(final LocalHistoryIdentifier historyId, + final int initialSerializedBufferCapacity) { + final ByteArrayDataOutput out = ByteStreams.newDataOutput(initialSerializedBufferCapacity); try { historyId.writeTo(out); } catch (IOException e) { diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeTransactionPayload.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeTransactionPayload.java index 3141d69f58..ac849723e1 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeTransactionPayload.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/persisted/PurgeTransactionPayload.java @@ -54,8 +54,9 @@ public final class PurgeTransactionPayload extends AbstractIdentifiablePayload