From f746b92d9aa9495a13ce92c01c5c1ae3228bb662 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Thu, 6 Jun 2019 19:36:06 +0200 Subject: [PATCH] Add QNameModule coding Coding on QNames is effective when there are a lot of same QNames, but there are cases where QNames themeselves are not reused -- for example YangInstanceIdentifiers and small read subtrees. These typically end up using different QNames from a small set of modules, hence coding modules actually helps them quite a bit. This patch adds a namespace/revision cache, so that encoding such QNames is also efficient. JIRA: CONTROLLER-1898 Change-Id: I6cd0885ef605e4a1bf7b768ffe215e9eb3e2fbc5 Signed-off-by: Robert Varga --- .../datastore/node/utils/QNameFactory.java | 112 ++++++++++++++++-- ...ithiumNormalizedNodeInputStreamReader.java | 5 +- ...thiumNormalizedNodeOutputStreamWriter.java | 9 +- ...SodiumNormalizedNodeInputStreamReader.java | 39 +++++- ...odiumNormalizedNodeOutputStreamWriter.java | 21 +++- .../node/utils/stream/TokenTypes.java | 5 +- .../node/utils/QNameFactoryTest.java | 9 +- 7 files changed, 172 insertions(+), 28 deletions(-) diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java index f1b51ce261..5c465e0e6f 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java @@ -12,19 +12,22 @@ import static java.util.Objects.requireNonNull; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import java.net.URI; import java.util.Objects; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; import org.opendaylight.yangtools.concepts.Immutable; import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.QNameModule; +import org.opendaylight.yangtools.yang.common.Revision; public final class QNameFactory { - public static final class Key implements Immutable { + private static final class StringQName implements Immutable { private final @NonNull String localName; private final @NonNull String namespace; private final @Nullable String revision; - public Key(final String localName, final String namespace, final String revision) { + StringQName(final String localName, final String namespace, final String revision) { this.localName = requireNonNull(localName); this.namespace = requireNonNull(namespace); this.revision = revision; @@ -40,10 +43,10 @@ public final class QNameFactory { if (this == obj) { return true; } - if (!(obj instanceof Key)) { + if (!(obj instanceof StringQName)) { return false; } - final Key other = (Key) obj; + final StringQName other = (StringQName) obj; return localName.equals(other.localName) && namespace.equals(other.namespace) && Objects.equals(revision, other.revision); } @@ -53,24 +56,101 @@ public final class QNameFactory { } } + private static final class ModuleQName implements Immutable { + private final @NonNull QNameModule module; + private final @NonNull String localName; + + ModuleQName(final QNameModule module, final String localName) { + this.module = requireNonNull(module); + this.localName = requireNonNull(localName); + } + + @Override + public int hashCode() { + return 31 * module.hashCode() + localName.hashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ModuleQName)) { + return false; + } + final ModuleQName other = (ModuleQName) obj; + return localName.equals(other.localName) && module.equals(other.module); + } + + QName toQName() { + return QName.create(module, localName); + } + } + + private static final class StringModule implements Immutable { + private final @NonNull String namespace; + private final @Nullable String revision; + + StringModule(final String namespace, final String revision) { + this.namespace = requireNonNull(namespace); + this.revision = revision; + } + + @Override + public int hashCode() { + return 31 * namespace.hashCode() + Objects.hashCode(revision); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof StringModule)) { + return false; + } + final StringModule other = (StringModule) obj; + return namespace.equals(other.namespace) && Objects.equals(revision, other.revision); + } + + QNameModule toQNameModule() { + return QNameModule.create(URI.create(namespace), Revision.ofNullable(revision)); + } + } + private static final int MAX_QNAME_CACHE_SIZE = Integer.getInteger( "org.opendaylight.controller.cluster.datastore.node.utils.qname-cache.max-size", 10000); + private static final int MAX_MODULE_CACHE_SIZE = Integer.getInteger( + "org.opendaylight.controller.cluster.datastore.node.utils.module-cache.max-size", 2000); - private static final LoadingCache STRING_CACHE = CacheBuilder.newBuilder() + private static final LoadingCache LEGACY_CACHE = CacheBuilder.newBuilder() .maximumSize(MAX_QNAME_CACHE_SIZE).weakValues().build(new CacheLoader() { @Override public QName load(final String key) { return QName.create(key).intern(); } }); - - private static final LoadingCache KEY_CACHE = CacheBuilder.newBuilder() - .maximumSize(MAX_QNAME_CACHE_SIZE).weakValues().build(new CacheLoader() { + private static final LoadingCache STRING_CACHE = CacheBuilder.newBuilder() + .maximumSize(MAX_QNAME_CACHE_SIZE).weakValues().build(new CacheLoader() { + @Override + public QName load(final StringQName key) { + return key.toQName().intern(); + } + }); + private static final LoadingCache QNAME_CACHE = CacheBuilder.newBuilder() + .maximumSize(MAX_QNAME_CACHE_SIZE).weakValues().build(new CacheLoader() { @Override - public QName load(final Key key) { + public QName load(final ModuleQName key) { return key.toQName().intern(); } }); + private static final LoadingCache MODULE_CACHE = CacheBuilder.newBuilder() + .maximumSize(MAX_MODULE_CACHE_SIZE).weakValues().build(new CacheLoader() { + @Override + public QNameModule load(final StringModule key) { + return key.toQNameModule().intern(); + } + }); private QNameFactory() { @@ -78,10 +158,18 @@ public final class QNameFactory { @Deprecated public static QName create(final String name) { - return STRING_CACHE.getUnchecked(name); + return LEGACY_CACHE.getUnchecked(name); + } + + public static QName create(final String localName, final String namespace, final @Nullable String revision) { + return STRING_CACHE.getUnchecked(new StringQName(localName, namespace, revision)); + } + + public static QName create(final QNameModule module, final String localName) { + return QNAME_CACHE.getUnchecked(new ModuleQName(module, localName)); } - public static QName create(final Key key) { - return KEY_CACHE.getUnchecked(key); + public static QNameModule createModule(final String namespace, final @Nullable String revision) { + return MODULE_CACHE.getUnchecked(new StringModule(namespace, revision)); } } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeInputStreamReader.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeInputStreamReader.java index 84e96bfb91..1484f735d0 100755 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeInputStreamReader.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeInputStreamReader.java @@ -214,11 +214,10 @@ class LithiumNormalizedNodeInputStreamReader extends ForwardingDataInput impleme String namespace = readCodedString(); String revision = Strings.emptyToNull(readCodedString()); - return QNameFactory.create(new QNameFactory.Key(localName, namespace, revision)); + return QNameFactory.create(localName, namespace, revision); } - - private String readCodedString() throws IOException { + final String readCodedString() throws IOException { final byte valueType = input.readByte(); switch (valueType) { case TokenTypes.IS_NULL_VALUE: diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeOutputStreamWriter.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeOutputStreamWriter.java index e777948ca5..a0aa813f64 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeOutputStreamWriter.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/LithiumNormalizedNodeOutputStreamWriter.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.QNameModule; import org.opendaylight.yangtools.yang.common.Revision; /** @@ -43,8 +44,12 @@ class LithiumNormalizedNodeOutputStreamWriter extends AbstractNormalizedNodeData @Override protected void writeQName(final QName qname) throws IOException { writeString(qname.getLocalName()); - writeString(qname.getNamespace().toString()); - writeString(qname.getRevision().map(Revision::toString).orElse(null)); + writeModule(qname.getModule()); + } + + void writeModule(final QNameModule module) throws IOException { + writeString(module.getNamespace().toString()); + writeString(module.getRevision().map(Revision::toString).orElse(null)); } @Override diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeInputStreamReader.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeInputStreamReader.java index 4890180115..fdc6adf225 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeInputStreamReader.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeInputStreamReader.java @@ -13,13 +13,16 @@ import java.io.DataInput; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.opendaylight.controller.cluster.datastore.node.utils.QNameFactory; import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.QNameModule; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.AugmentationIdentifier; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier; final class SodiumNormalizedNodeInputStreamReader extends LithiumNormalizedNodeInputStreamReader { private final ArrayList codedNodeIdentifiers = new ArrayList<>(); private final List codedAugments = new ArrayList<>(); + private final List codedModules = new ArrayList<>(); private final List codedQNames = new ArrayList<>(); SodiumNormalizedNodeInputStreamReader(final DataInput input) { @@ -49,7 +52,7 @@ final class SodiumNormalizedNodeInputStreamReader extends LithiumNormalizedNodeI final byte valueType = readByte(); switch (valueType) { case TokenTypes.IS_AUGMENT_CODE: - return codecAugmentId(readInt()); + return codedAugmentId(readInt()); case TokenTypes.IS_AUGMENT_VALUE: return rawAugmentId(); default: @@ -72,6 +75,18 @@ final class SodiumNormalizedNodeInputStreamReader extends LithiumNormalizedNodeI } } + private QNameModule readModule() throws IOException { + final byte valueType = readByte(); + switch (valueType) { + case TokenTypes.IS_MODULE_CODE: + return codedModule(readInt()); + case TokenTypes.IS_MODULE_VALUE: + return rawModule(); + default: + throw new IOException("Unhandled QName value type " + valueType); + } + } + private NodeIdentifier codedNodeIdentifier(final int code) throws IOException { final NodeIdentifier existing = codedNodeIdentifiers.size() > code ? codedNodeIdentifiers.get(code) : null; return existing != null ? existing : storeNodeIdentifier(code, codedQName(code)); @@ -112,12 +127,14 @@ final class SodiumNormalizedNodeInputStreamReader extends LithiumNormalizedNodeI } private QName rawQName() throws IOException { - final QName qname = super.readQName(); + final String localName = readCodedString(); + final QNameModule module = readModule(); + final QName qname = QNameFactory.create(module, localName); codedQNames.add(qname); return qname; } - private AugmentationIdentifier codecAugmentId(final int code) throws IOException { + private AugmentationIdentifier codedAugmentId(final int code) throws IOException { try { return codedAugments.get(code); } catch (IndexOutOfBoundsException e) { @@ -130,4 +147,20 @@ final class SodiumNormalizedNodeInputStreamReader extends LithiumNormalizedNodeI codedAugments.add(aid); return aid; } + + private QNameModule codedModule(final int code) throws IOException { + try { + return codedModules.get(code); + } catch (IndexOutOfBoundsException e) { + throw new IOException("Module code " + code + " was not found", e); + } + } + + private QNameModule rawModule() throws IOException { + final String namespace = readCodedString(); + final String revision = readCodedString(); + final QNameModule mod = QNameFactory.createModule(namespace, revision); + codedModules.add(mod); + return mod; + } } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeOutputStreamWriter.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeOutputStreamWriter.java index c549290f81..5f5c0a01fd 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeOutputStreamWriter.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumNormalizedNodeOutputStreamWriter.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.QNameModule; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.AugmentationIdentifier; /** @@ -30,6 +31,7 @@ import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.Augmentat */ class SodiumNormalizedNodeOutputStreamWriter extends LithiumNormalizedNodeOutputStreamWriter { private final Map aidCodeMap = new HashMap<>(); + private final Map moduleCodeMap = new HashMap<>(); private final Map qnameCodeMap = new HashMap<>(); SodiumNormalizedNodeOutputStreamWriter(final DataOutput output) { @@ -60,14 +62,29 @@ class SodiumNormalizedNodeOutputStreamWriter extends LithiumNormalizedNodeOutput void writeAugmentationIdentifier(final AugmentationIdentifier aid) throws IOException { final Integer value = aidCodeMap.get(aid); if (value == null) { - // Fresh QName, remember it and emit as three strings + // Fresh AugmentationIdentifier, remember it and emit as three strings aidCodeMap.put(aid, aidCodeMap.size()); writeByte(TokenTypes.IS_AUGMENT_VALUE); super.writeAugmentationIdentifier(aid); } else { - // We have already seen this QName set: write its code + // We have already seen this AugmentationIdentifier: write its code writeByte(TokenTypes.IS_AUGMENT_CODE); writeInt(value); } } + + @Override + void writeModule(final QNameModule module) throws IOException { + final Integer value = moduleCodeMap.get(module); + if (value == null) { + // Fresh QNameModule, remember it and emit as three strings + moduleCodeMap.put(module, moduleCodeMap.size()); + writeByte(TokenTypes.IS_MODULE_VALUE); + super.writeModule(module); + } else { + // We have already seen this QNameModule: write its code + writeByte(TokenTypes.IS_MODULE_CODE); + writeInt(value); + } + } } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java index d41a4f0011..5fa9ce97b9 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java @@ -19,7 +19,8 @@ final class TokenTypes { */ static final short LITHIUM_VERSION = 1; /** - * Revised stream version. Unlike {@link #LITHIUM_VERSION}, QNames are using a per-stream dictionary, too. + * Revised stream version. Unlike {@link #LITHIUM_VERSION}, QNames and QNameModules are using a per-stream + * dictionary, too. */ static final short SODIUM_VERSION = 2; @@ -33,4 +34,6 @@ final class TokenTypes { static final byte IS_QNAME_VALUE = 5; static final byte IS_AUGMENT_CODE = 6; static final byte IS_AUGMENT_VALUE = 7; + static final byte IS_MODULE_CODE = 8; + static final byte IS_MODULE_VALUE = 9; } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/test/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactoryTest.java b/opendaylight/md-sal/sal-clustering-commons/src/test/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactoryTest.java index 9f5419d203..557675edb9 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/test/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactoryTest.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/test/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactoryTest.java @@ -12,7 +12,6 @@ import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import org.junit.Test; -import org.opendaylight.controller.cluster.datastore.node.utils.QNameFactory.Key; import org.opendaylight.controller.cluster.datastore.util.TestModel; import org.opendaylight.yangtools.yang.common.QName; import org.opendaylight.yangtools.yang.common.Revision; @@ -34,16 +33,16 @@ public class QNameFactoryTest { @Test public void testBasic() { QName expected = TestModel.AUG_NAME_QNAME; - QName created = QNameFactory.create(createKey(expected)); + QName created = lookup(expected); assertNotSame(expected, created); assertEquals(expected, created); - QName cached = QNameFactory.create(createKey(expected)); + QName cached = lookup(expected); assertSame(created, cached); } - private static Key createKey(final QName qname) { - return new Key(qname.getLocalName(), qname.getNamespace().toString(), + private static QName lookup(final QName qname) { + return QNameFactory.create(qname.getLocalName(), qname.getNamespace().toString(), qname.getRevision().map(Revision::toString).orElse(null)); } } -- 2.36.6