From 43ab81083ba4ffdbf6999df1e6fe8750ce2c76ac Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Mon, 9 Sep 2019 10:29:57 +0200 Subject: [PATCH] Add LithiumSR1 input/output support This is a code drop of both DataInput and DataOutput for Sodium SR1 streaming format. It uses its own dedicated tokes, along with quite a few of stateful encoding rules. Where Lithium did encoding did not perform any look-behind except for LeafSetEntryNodes and only maintained simple coding tables, this coder maintains simple 'parent' state tracking. While the state tracking is not mandated on the encoder side, some amount of tracking is required in the decoder side to provide full coverage of coding constructs. Two examples of this are: - parent QName references, which are expanded to cover not only LeafSet/LeafSetEntry, but also UnkeyedList/UnkeyedListEntry and most importantly Map/MapEntry pairs - Map/MapEntry tracking is also important for coding leaf nodes which represent keys -- in those cases emitting the leaf value is superfluous, as it can be looked up from MapEntry's predicates Also this state tracking is maintained on-stack, thus making issues related with it nigh impossible (unlike the Lithium LeafSetEntry bugs). JIRA: CONTROLLER-1919 Change-Id: I5fcf85a76750301cd3f8bcbd505baa0f95397cb5 Signed-off-by: Robert Varga --- .../datastore/node/utils/QNameFactory.java | 14 + .../stream/AbstractMagnesiumDataInput.java | 845 ++++++++++++++++++ .../stream/AbstractMagnesiumDataOutput.java | 690 ++++++++++++++ .../stream/NormalizedNodeInputOutput.java | 2 + .../stream/NormalizedNodeStreamVersion.java | 3 +- .../node/utils/stream/SodiumSR1DataInput.java | 29 + .../utils/stream/SodiumSR1DataOutput.java | 29 + .../node/utils/stream/TokenTypes.java | 4 + .../VersionedNormalizedNodeDataInput.java | 3 + 9 files changed, 1618 insertions(+), 1 deletion(-) create mode 100644 opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataInput.java create mode 100644 opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataOutput.java create mode 100644 opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataInput.java create mode 100644 opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataOutput.java diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java index 5c465e0e6f..a0d8a311f7 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/QNameFactory.java @@ -14,12 +14,14 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import java.net.URI; import java.util.Objects; +import java.util.concurrent.ExecutionException; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; import org.opendaylight.yangtools.concepts.Immutable; import org.opendaylight.yangtools.yang.common.QName; import org.opendaylight.yangtools.yang.common.QNameModule; import org.opendaylight.yangtools.yang.common.Revision; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier; public final class QNameFactory { private static final class StringQName implements Immutable { @@ -151,6 +153,13 @@ public final class QNameFactory { return key.toQNameModule().intern(); } }); + private static final LoadingCache NODEID_CACHE = CacheBuilder.newBuilder() + .maximumSize(MAX_QNAME_CACHE_SIZE).weakValues().build(new CacheLoader() { + @Override + public NodeIdentifier load(final ModuleQName key) throws ExecutionException { + return NodeIdentifier.create(QNAME_CACHE.get(key)); + } + }); private QNameFactory() { @@ -172,4 +181,9 @@ public final class QNameFactory { public static QNameModule createModule(final String namespace, final @Nullable String revision) { return MODULE_CACHE.getUnchecked(new StringModule(namespace, revision)); } + + public static @NonNull NodeIdentifier getNodeIdentifier(final QNameModule module, final String localName) + throws ExecutionException { + return NODEID_CACHE.get(new ModuleQName(module, localName)); + } } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataInput.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataInput.java new file mode 100644 index 0000000000..2d02dd11ca --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataInput.java @@ -0,0 +1,845 @@ +/* + * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.node.utils.stream; + +import static com.google.common.base.Verify.verifyNotNull; +import static java.util.Objects.requireNonNull; +import static org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter.UNKNOWN_SIZE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.UncheckedExecutionException; +import java.io.DataInput; +import java.io.IOException; +import java.io.StringReader; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import javax.xml.transform.dom.DOMSource; +import org.eclipse.jdt.annotation.NonNull; +import org.opendaylight.controller.cluster.datastore.node.utils.QNameFactory; +import org.opendaylight.yangtools.rfc8528.data.api.MountPointIdentifier; +import org.opendaylight.yangtools.util.xml.UntrustedXML; +import org.opendaylight.yangtools.yang.common.Empty; +import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.QNameModule; +import org.opendaylight.yangtools.yang.common.Uint16; +import org.opendaylight.yangtools.yang.common.Uint32; +import org.opendaylight.yangtools.yang.common.Uint64; +import org.opendaylight.yangtools.yang.common.Uint8; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.AugmentationIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeWithValue; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +/** + * Abstract base class for NormalizedNodeDataInput based on {@link MagnesiumNode}, {@link MagnesiumPathArgument} and + * {@link MagnesiumValue}. + */ +abstract class AbstractMagnesiumDataInput extends AbstractNormalizedNodeDataInput { + private static final Logger LOG = LoggerFactory.getLogger(AbstractMagnesiumDataInput.class); + + // Known singleton objects + private static final @NonNull Byte INT8_0 = 0; + private static final @NonNull Short INT16_0 = 0; + private static final @NonNull Integer INT32_0 = 0; + private static final @NonNull Long INT64_0 = 0L; + private static final byte @NonNull[] BINARY_0 = new byte[0]; + private static final @NonNull AugmentationIdentifier EMPTY_AID = AugmentationIdentifier.create(ImmutableSet.of()); + + // FIXME: these should be available as constants + private static final @NonNull Uint8 UINT8_0 = Uint8.valueOf(0); + private static final @NonNull Uint16 UINT16_0 = Uint16.valueOf(0); + private static final @NonNull Uint32 UINT32_0 = Uint32.valueOf(0); + private static final @NonNull Uint64 UINT64_0 = Uint64.valueOf(0); + + private final List codedAugments = new ArrayList<>(); + private final List codedNodeIdentifiers = new ArrayList<>(); + private final List codedModules = new ArrayList<>(); + private final List codedStrings = new ArrayList<>(); + + AbstractMagnesiumDataInput(final DataInput input) { + super(input); + } + + @Override + public final void streamNormalizedNode(final NormalizedNodeStreamWriter writer) throws IOException { + streamNormalizedNode(requireNonNull(writer), null, input.readByte()); + } + + private void streamNormalizedNode(final NormalizedNodeStreamWriter writer, final PathArgument parent, + final byte nodeHeader) throws IOException { + switch (nodeHeader & MagnesiumNode.TYPE_MASK) { + case MagnesiumNode.NODE_LEAF: + streamLeaf(writer, parent, nodeHeader); + break; + case MagnesiumNode.NODE_CONTAINER: + streamContainer(writer, nodeHeader); + break; + case MagnesiumNode.NODE_LIST: + streamList(writer, nodeHeader); + break; + case MagnesiumNode.NODE_MAP: + streamMap(writer, nodeHeader); + break; + case MagnesiumNode.NODE_MAP_ORDERED: + streamMapOrdered(writer, nodeHeader); + break; + case MagnesiumNode.NODE_LEAFSET: + streamLeafset(writer, nodeHeader); + break; + case MagnesiumNode.NODE_LEAFSET_ORDERED: + streamLeafsetOrdered(writer, nodeHeader); + break; + case MagnesiumNode.NODE_CHOICE: + streamChoice(writer, nodeHeader); + break; + case MagnesiumNode.NODE_AUGMENTATION: + streamAugmentation(writer, nodeHeader); + break; + case MagnesiumNode.NODE_ANYXML: + streamAnyxml(writer, nodeHeader); + break; + case MagnesiumNode.NODE_ANYXML_MODELED: + streamAnyxmlModeled(writer, nodeHeader); + break; + case MagnesiumNode.NODE_LIST_ENTRY: + streamListEntry(writer, parent, nodeHeader); + break; + case MagnesiumNode.NODE_LEAFSET_ENTRY: + streamLeafsetEntry(writer, parent, nodeHeader); + break; + case MagnesiumNode.NODE_MAP_ENTRY: + streamMapEntry(writer, parent, nodeHeader); + break; + default: + throw new InvalidNormalizedNodeStreamException("Unexpected node header " + nodeHeader); + } + } + + private void streamAnyxml(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming anyxml node {}", identifier); + writer.startAnyxmlNode(identifier); + writer.domSourceValue(readDOMSource()); + writer.endNode(); + } + + private void streamAnyxmlModeled(final NormalizedNodeStreamWriter writer, final byte nodeHeader) + throws IOException { + // TODO: decide how to deal with these + throw new UnsupportedOperationException("Reading YANG-modeled anyxml was never supported"); + } + + private void streamAugmentation(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final AugmentationIdentifier augIdentifier = decodeAugmentationIdentifier(nodeHeader); + LOG.trace("Streaming augmentation node {}", augIdentifier); + writer.startAugmentationNode(augIdentifier); + commonStreamContainer(writer, augIdentifier); + } + + private void streamChoice(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming choice node {}", identifier); + writer.startChoiceNode(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamContainer(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming container node {}", identifier); + writer.startContainerNode(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamLeaf(final NormalizedNodeStreamWriter writer, final PathArgument parent, final byte nodeHeader) + throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming leaf node {}", identifier); + writer.startLeafNode(identifier); + + final Object value; + if ((nodeHeader & MagnesiumNode.PREDICATE_ONE) == MagnesiumNode.PREDICATE_ONE) { + if (!(parent instanceof NodeIdentifierWithPredicates)) { + throw new InvalidNormalizedNodeStreamException("Invalid predicate leaf " + identifier + " in parent " + + parent); + } + + value = ((NodeIdentifierWithPredicates) parent).getValue(identifier.getNodeType()); + if (value == null) { + throw new InvalidNormalizedNodeStreamException("Failed to find predicate leaf " + identifier + + " in parent " + parent); + } + } else { + value = readLeafValue(); + } + + writer.scalarValue(value); + writer.endNode(); + } + + private void streamLeafset(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming leaf set node {}", identifier); + writer.startLeafSet(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamLeafsetOrdered(final NormalizedNodeStreamWriter writer, final byte nodeHeader) + throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming ordered leaf set node {}", identifier); + writer.startOrderedLeafSet(identifier, UNKNOWN_SIZE); + + commonStreamContainer(writer, identifier); + } + + private void streamLeafsetEntry(final NormalizedNodeStreamWriter writer, final PathArgument parent, + final byte nodeHeader) throws IOException { + final NodeIdentifier nodeId = decodeNodeIdentifier(nodeHeader, parent); + final Object value = readLeafValue(); + final NodeWithValue leafIdentifier = new NodeWithValue<>(nodeId.getNodeType(), value); + LOG.trace("Streaming leaf set entry node {}", leafIdentifier); + writer.startLeafSetEntryNode(leafIdentifier); + writer.scalarValue(value); + writer.endNode(); + } + + private void streamList(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + writer.startUnkeyedList(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamListEntry(final NormalizedNodeStreamWriter writer, final PathArgument parent, + final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader, parent); + LOG.trace("Streaming unkeyed list item node {}", identifier); + writer.startUnkeyedListItem(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamMap(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming map node {}", identifier); + writer.startMapNode(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamMapOrdered(final NormalizedNodeStreamWriter writer, final byte nodeHeader) throws IOException { + final NodeIdentifier identifier = decodeNodeIdentifier(nodeHeader); + LOG.trace("Streaming ordered map node {}", identifier); + writer.startOrderedMapNode(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void streamMapEntry(final NormalizedNodeStreamWriter writer, final PathArgument parent, + final byte nodeHeader) throws IOException { + final NodeIdentifier nodeId = decodeNodeIdentifier(nodeHeader, parent); + + final int size; + switch (nodeHeader & MagnesiumNode.PREDICATE_MASK) { + case MagnesiumNode.PREDICATE_ZERO: + size = 0; + break; + case MagnesiumNode.PREDICATE_ONE: + size = 1; + break; + case MagnesiumNode.PREDICATE_1B: + size = input.readUnsignedByte(); + break; + case MagnesiumNode.PREDICATE_4B: + size = input.readInt(); + break; + default: + // ISE on purpose: this should never ever happen + throw new IllegalStateException("Failed to decode NodeIdentifierWithPredicates size from header " + + nodeHeader); + } + + final NodeIdentifierWithPredicates identifier = readNodeIdentifierWithPredicates(nodeId.getNodeType(), size); + LOG.trace("Streaming map entry node {}", identifier); + writer.startMapEntryNode(identifier, UNKNOWN_SIZE); + commonStreamContainer(writer, identifier); + } + + private void commonStreamContainer(final NormalizedNodeStreamWriter writer, final PathArgument parent) + throws IOException { + for (byte nodeType = input.readByte(); nodeType != MagnesiumNode.NODE_END; nodeType = input.readByte()) { + streamNormalizedNode(writer, parent, nodeType); + } + writer.endNode(); + } + + private @NonNull NodeIdentifier decodeNodeIdentifier() throws IOException { + final QNameModule module = decodeQNameModule(); + final String localName = readRefString(); + final NodeIdentifier nodeId; + try { + nodeId = QNameFactory.getNodeIdentifier(module, localName); + } catch (ExecutionException e) { + throw new InvalidNormalizedNodeStreamException("Illegal QName module=" + module + " localName=" + + localName, e); + } + + codedNodeIdentifiers.add(nodeId); + return nodeId; + } + + private NodeIdentifier decodeNodeIdentifier(final byte nodeHeader) throws IOException { + return decodeNodeIdentifier(nodeHeader, null); + } + + private NodeIdentifier decodeNodeIdentifier(final byte nodeHeader, final PathArgument parent) throws IOException { + final int index; + switch (nodeHeader & MagnesiumNode.ADDR_MASK) { + case MagnesiumNode.ADDR_DEFINE: + return readNodeIdentifier(); + case MagnesiumNode.ADDR_LOOKUP_1B: + index = input.readUnsignedByte(); + break; + case MagnesiumNode.ADDR_LOOKUP_4B: + index = input.readInt(); + break; + case MagnesiumNode.ADDR_PARENT: + if (parent instanceof NodeIdentifier) { + return (NodeIdentifier) parent; + } + throw new InvalidNormalizedNodeStreamException("Invalid node identifier reference to parent " + parent); + default: + throw new InvalidNormalizedNodeStreamException("Unexpected node identifier addressing in header " + + nodeHeader); + } + + try { + return codedNodeIdentifiers.get(index); + } catch (IndexOutOfBoundsException e) { + throw new InvalidNormalizedNodeStreamException("Invalid QName reference " + index, e); + } + } + + private AugmentationIdentifier decodeAugmentationIdentifier(final byte nodeHeader) throws IOException { + final int index; + switch (nodeHeader & MagnesiumNode.ADDR_MASK) { + case MagnesiumNode.ADDR_DEFINE: + return readAugmentationIdentifier(); + case MagnesiumNode.ADDR_LOOKUP_1B: + index = input.readUnsignedByte(); + break; + case MagnesiumNode.ADDR_LOOKUP_4B: + index = input.readInt(); + break; + default: + throw new InvalidNormalizedNodeStreamException( + "Unexpected augmentation identifier addressing in header " + nodeHeader); + } + + try { + return codedAugments.get(index); + } catch (IndexOutOfBoundsException e) { + throw new InvalidNormalizedNodeStreamException("Invalid augmentation identifier reference " + index, e); + } + } + + @Override + public final YangInstanceIdentifier readYangInstanceIdentifier() throws IOException { + final byte type = input.readByte(); + if (type == MagnesiumValue.YIID) { + return readYangInstanceIdentifier(input.readInt()); + } else if (type >= MagnesiumValue.YIID_0) { + // Note 'byte' is range limited, so it is always '&& type <= MagnesiumValue.YIID_31' + return readYangInstanceIdentifier(type - MagnesiumValue.YIID_0); + } else { + throw new InvalidNormalizedNodeStreamException("Unexpected YangInstanceIdentifier type " + type); + } + } + + private @NonNull YangInstanceIdentifier readYangInstanceIdentifier(final int size) throws IOException { + if (size > 0) { + final Builder builder = ImmutableList.builderWithExpectedSize(size); + for (int i = 0; i < size; ++i) { + builder.add(readPathArgument()); + } + return YangInstanceIdentifier.create(builder.build()); + } else if (size == 0) { + return YangInstanceIdentifier.EMPTY; + } else { + throw new InvalidNormalizedNodeStreamException("Invalid YangInstanceIdentifier size " + size); + } + } + + @Override + public final QName readQName() throws IOException { + final byte type = input.readByte(); + switch (type) { + case MagnesiumValue.QNAME: + return decodeQName(); + case MagnesiumValue.QNAME_REF_1B: + return decodeQNameRef1(); + case MagnesiumValue.QNAME_REF_2B: + return decodeQNameRef2(); + case MagnesiumValue.QNAME_REF_4B: + return decodeQNameRef4(); + default: + throw new InvalidNormalizedNodeStreamException("Unexpected QName type " + type); + } + } + + @Override + public final PathArgument readPathArgument() throws IOException { + final byte header = input.readByte(); + switch (header & MagnesiumPathArgument.TYPE_MASK) { + case MagnesiumPathArgument.AUGMENTATION_IDENTIFIER: + return readAugmentationIdentifier(header); + case MagnesiumPathArgument.NODE_IDENTIFIER: + verifyPathIdentifierOnly(header); + return readNodeIdentifier(header); + case MagnesiumPathArgument.NODE_IDENTIFIER_WITH_PREDICATES: + return readNodeIdentifierWithPredicates(header); + case MagnesiumPathArgument.NODE_WITH_VALUE: + verifyPathIdentifierOnly(header); + return readNodeWithValue(header); + case MagnesiumPathArgument.MOUNTPOINT_IDENTIFIER: + verifyPathIdentifierOnly(header); + return MountPointIdentifier.create(readNodeIdentifier(header).getNodeType()); + default: + throw new InvalidNormalizedNodeStreamException("Unexpected PathArgument header " + header); + } + } + + private AugmentationIdentifier readAugmentationIdentifier() throws IOException { + final AugmentationIdentifier result = readAugmentationIdentifier(input.readInt()); + codedAugments.add(result); + return result; + } + + private AugmentationIdentifier readAugmentationIdentifier(final byte header) throws IOException { + final int count = header & MagnesiumPathArgument.AID_COUNT_MASK; + switch (count) { + case MagnesiumPathArgument.AID_COUNT_1B: + return readAugmentationIdentifier(input.readUnsignedByte()); + case MagnesiumPathArgument.AID_COUNT_2B: + return readAugmentationIdentifier(input.readUnsignedShort()); + case MagnesiumPathArgument.AID_COUNT_4B: + return readAugmentationIdentifier(input.readInt()); + default: + return readAugmentationIdentifier(count >>> MagnesiumPathArgument.AID_COUNT_SHIFT); + } + } + + private AugmentationIdentifier readAugmentationIdentifier(final int size) throws IOException { + if (size > 0) { + final List qnames = new ArrayList<>(size); + for (int i = 0; i < size; ++i) { + qnames.add(readQName()); + } + return AugmentationIdentifier.create(ImmutableSet.copyOf(qnames)); + } else if (size == 0) { + return EMPTY_AID; + } else { + throw new InvalidNormalizedNodeStreamException("Invalid augmentation identifier size " + size); + } + } + + private NodeIdentifier readNodeIdentifier() throws IOException { + return decodeNodeIdentifier(); + } + + private NodeIdentifier readNodeIdentifier(final byte header) throws IOException { + switch (header & MagnesiumPathArgument.QNAME_MASK) { + case MagnesiumPathArgument.QNAME_DEF: + return decodeNodeIdentifier(); + case MagnesiumPathArgument.QNAME_REF_1B: + return decodeNodeIdentifierRef1(); + case MagnesiumPathArgument.QNAME_REF_2B: + return decodeNodeIdentifierRef2(); + case MagnesiumPathArgument.QNAME_REF_4B: + return decodeNodeIdentifierRef4(); + default: + throw new InvalidNormalizedNodeStreamException("Invalid QName coding in " + header); + } + } + + private NodeIdentifierWithPredicates readNodeIdentifierWithPredicates(final byte header) throws IOException { + final QName qname = readNodeIdentifier(header).getNodeType(); + switch (header & MagnesiumPathArgument.SIZE_MASK) { + case MagnesiumPathArgument.SIZE_1B: + return readNodeIdentifierWithPredicates(qname, input.readUnsignedByte()); + case MagnesiumPathArgument.SIZE_2B: + return readNodeIdentifierWithPredicates(qname, input.readUnsignedShort()); + case MagnesiumPathArgument.SIZE_4B: + return readNodeIdentifierWithPredicates(qname, input.readInt()); + default: + return readNodeIdentifierWithPredicates(qname, header >>> MagnesiumPathArgument.SIZE_SHIFT); + } + } + + private NodeIdentifierWithPredicates readNodeIdentifierWithPredicates(final QName qname, final int size) + throws IOException { + if (size == 1) { + return NodeIdentifierWithPredicates.of(qname, readQName(), readLeafValue()); + } else if (size > 1) { + final ImmutableMap.Builder builder = ImmutableMap.builderWithExpectedSize(size); + for (int i = 0; i < size; ++i) { + builder.put(readQName(), readLeafValue()); + } + return NodeIdentifierWithPredicates.of(qname, builder.build()); + } else if (size == 0) { + return NodeIdentifierWithPredicates.of(qname); + } else { + throw new InvalidNormalizedNodeStreamException("Invalid predicate count " + size); + } + } + + private NodeWithValue readNodeWithValue(final byte header) throws IOException { + final QName qname = readNodeIdentifier(header).getNodeType(); + return new NodeWithValue<>(qname, readLeafValue()); + } + + private static void verifyPathIdentifierOnly(final byte header) throws InvalidNormalizedNodeStreamException { + if ((header & MagnesiumPathArgument.SIZE_MASK) != 0) { + throw new InvalidNormalizedNodeStreamException("Invalid path argument header " + header); + } + } + + private @NonNull NodeIdentifier decodeNodeIdentifierRef1() throws IOException { + return lookupNodeIdentifier(input.readUnsignedByte()); + } + + private @NonNull NodeIdentifier decodeNodeIdentifierRef2() throws IOException { + return lookupNodeIdentifier(input.readUnsignedShort() + 256); + } + + private @NonNull NodeIdentifier decodeNodeIdentifierRef4() throws IOException { + return lookupNodeIdentifier(input.readInt()); + } + + private @NonNull QName decodeQName() throws IOException { + return decodeNodeIdentifier().getNodeType(); + } + + private @NonNull QName decodeQNameRef1() throws IOException { + return lookupQName(input.readUnsignedByte()); + } + + private @NonNull QName decodeQNameRef2() throws IOException { + return lookupQName(input.readUnsignedShort() + 256); + } + + private @NonNull QName decodeQNameRef4() throws IOException { + return lookupQName(input.readInt()); + } + + private @NonNull QNameModule decodeQNameModule() throws IOException { + final byte type = input.readByte(); + final int index; + switch (type) { + case MagnesiumValue.MODREF_1B: + index = input.readUnsignedByte(); + break; + case MagnesiumValue.MODREF_2B: + index = input.readUnsignedShort() + 256; + break; + case MagnesiumValue.MODREF_4B: + index = input.readInt(); + break; + default: + return decodeQNameModuleDef(type); + } + + try { + return codedModules.get(index); + } catch (IndexOutOfBoundsException e) { + throw new InvalidNormalizedNodeStreamException("Invalid QNameModule reference " + index, e); + } + } + + // QNameModule definition, i.e. two encoded strings + private @NonNull QNameModule decodeQNameModuleDef(final byte type) throws IOException { + final String namespace = readRefString(type); + + final byte refType = input.readByte(); + final String revision = refType == MagnesiumValue.STRING_EMPTY ? null : readRefString(refType); + final QNameModule module; + try { + module = QNameFactory.createModule(namespace, revision); + } catch (UncheckedExecutionException e) { + throw new InvalidNormalizedNodeStreamException("Illegal QNameModule ns=" + namespace + " rev=" + revision, + e); + } + + codedModules.add(module); + return module; + } + + private @NonNull String readRefString() throws IOException { + return readRefString(input.readByte()); + } + + private @NonNull String readRefString(final byte type) throws IOException { + final String str; + switch (type) { + case MagnesiumValue.STRING_REF_1B: + return lookupString(input.readUnsignedByte()); + case MagnesiumValue.STRING_REF_2B: + return lookupString(input.readUnsignedShort() + 256); + case MagnesiumValue.STRING_REF_4B: + return lookupString(input.readInt()); + case MagnesiumValue.STRING_EMPTY: + return ""; + case MagnesiumValue.STRING_2B: + str = readString2(); + break; + case MagnesiumValue.STRING_4B: + str = readString4(); + break; + case MagnesiumValue.STRING_CHARS: + str = readCharsString(); + break; + case MagnesiumValue.STRING_UTF: + str = input.readUTF(); + break; + default: + throw new InvalidNormalizedNodeStreamException("Unexpected String type " + type); + } + + // TODO: consider interning Strings -- that would help with bits, but otherwise it's probably not worth it + codedStrings.add(verifyNotNull(str)); + return str; + } + + private @NonNull String readString() throws IOException { + final byte type = input.readByte(); + switch (type) { + case MagnesiumValue.STRING_EMPTY: + return ""; + case MagnesiumValue.STRING_UTF: + return input.readUTF(); + case MagnesiumValue.STRING_2B: + return readString2(); + case MagnesiumValue.STRING_4B: + return readString4(); + case MagnesiumValue.STRING_CHARS: + return readCharsString(); + default: + throw new InvalidNormalizedNodeStreamException("Unexpected String type " + type); + } + } + + private @NonNull String readString2() throws IOException { + return readByteString(input.readUnsignedShort()); + } + + private @NonNull String readString4() throws IOException { + return readByteString(input.readInt()); + } + + private @NonNull String readByteString(final int size) throws IOException { + if (size > 0) { + final byte[] bytes = new byte[size]; + input.readFully(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } else if (size == 0) { + return ""; + } else { + throw new InvalidNormalizedNodeStreamException("Invalid String bytes length " + size); + } + } + + private @NonNull String readCharsString() throws IOException { + final int size = input.readInt(); + if (size > 0) { + final char[] chars = new char[size]; + for (int i = 0; i < size; ++i) { + chars[i] = input.readChar(); + } + return String.valueOf(chars); + } else if (size == 0) { + return ""; + } else { + throw new InvalidNormalizedNodeStreamException("Invalid String chars length " + size); + } + } + + private @NonNull NodeIdentifier lookupNodeIdentifier(final int index) throws InvalidNormalizedNodeStreamException { + try { + return codedNodeIdentifiers.get(index); + } catch (IndexOutOfBoundsException e) { + throw new InvalidNormalizedNodeStreamException("Invalid QName reference " + index, e); + } + } + + private @NonNull QName lookupQName(final int index) throws InvalidNormalizedNodeStreamException { + return lookupNodeIdentifier(index).getNodeType(); + } + + private @NonNull String lookupString(final int index) throws InvalidNormalizedNodeStreamException { + try { + return codedStrings.get(index); + } catch (IndexOutOfBoundsException e) { + throw new InvalidNormalizedNodeStreamException("Invalid String reference " + index, e); + } + } + + private @NonNull DOMSource readDOMSource() throws IOException { + final String str = readString(); + try { + return new DOMSource(UntrustedXML.newDocumentBuilder().parse(new InputSource(new StringReader(str))) + .getDocumentElement()); + } catch (SAXException e) { + throw new IOException("Error parsing XML: " + str, e); + } + } + + private @NonNull Object readLeafValue() throws IOException { + final byte type = input.readByte(); + switch (type) { + case MagnesiumValue.BOOLEAN_FALSE: + return Boolean.FALSE; + case MagnesiumValue.BOOLEAN_TRUE: + return Boolean.TRUE; + case MagnesiumValue.EMPTY: + return Empty.getInstance(); + case MagnesiumValue.INT8: + return input.readByte(); + case MagnesiumValue.INT8_0: + return INT8_0; + case MagnesiumValue.INT16: + return input.readShort(); + case MagnesiumValue.INT16_0: + return INT16_0; + case MagnesiumValue.INT32: + return input.readInt(); + case MagnesiumValue.INT32_0: + return INT32_0; + case MagnesiumValue.INT32_2B: + return input.readShort() & 0xFFFF; + case MagnesiumValue.INT64: + return input.readLong(); + case MagnesiumValue.INT64_0: + return INT64_0; + case MagnesiumValue.INT64_4B: + return input.readInt() & 0xFFFFFFFFL; + case MagnesiumValue.UINT8: + return Uint8.fromByteBits(input.readByte()); + case MagnesiumValue.UINT8_0: + return UINT8_0; + case MagnesiumValue.UINT16: + return Uint16.fromShortBits(input.readShort()); + case MagnesiumValue.UINT16_0: + return UINT16_0; + case MagnesiumValue.UINT32: + return Uint32.fromIntBits(input.readInt()); + case MagnesiumValue.UINT32_0: + return UINT32_0; + case MagnesiumValue.UINT32_2B: + return Uint32.fromIntBits(input.readShort() & 0xFFFF); + case MagnesiumValue.UINT64: + return Uint64.fromLongBits(input.readLong()); + case MagnesiumValue.UINT64_0: + return UINT64_0; + case MagnesiumValue.UINT64_4B: + return Uint64.fromLongBits(input.readInt() & 0xFFFFFFFFL); + case MagnesiumValue.BIGDECIMAL: + // FIXME: use string -> BigDecimal cache + return new BigDecimal(input.readUTF()); + case MagnesiumValue.BIGINTEGER: + return readBigInteger(); + case MagnesiumValue.STRING_EMPTY: + return ""; + case MagnesiumValue.STRING_UTF: + return input.readUTF(); + case MagnesiumValue.STRING_2B: + return readString2(); + case MagnesiumValue.STRING_4B: + return readString4(); + case MagnesiumValue.STRING_CHARS: + return readCharsString(); + case MagnesiumValue.BINARY_0: + return BINARY_0; + case MagnesiumValue.BINARY_1B: + return readBinary(128 + input.readUnsignedByte()); + case MagnesiumValue.BINARY_2B: + return readBinary(384 + input.readUnsignedShort()); + case MagnesiumValue.BINARY_4B: + return readBinary(input.readInt()); + case MagnesiumValue.YIID_0: + return YangInstanceIdentifier.EMPTY; + case MagnesiumValue.YIID: + return readYangInstanceIdentifier(input.readInt()); + case MagnesiumValue.QNAME: + return decodeQName(); + case MagnesiumValue.QNAME_REF_1B: + return decodeQNameRef1(); + case MagnesiumValue.QNAME_REF_2B: + return decodeQNameRef2(); + case MagnesiumValue.QNAME_REF_4B: + return decodeQNameRef4(); + case MagnesiumValue.BITS_0: + return ImmutableSet.of(); + case MagnesiumValue.BITS_1B: + return readBits(input.readUnsignedByte() + 29); + case MagnesiumValue.BITS_2B: + return readBits(input.readUnsignedShort() + 285); + case MagnesiumValue.BITS_4B: + return readBits(input.readInt()); + + default: + if (type > MagnesiumValue.BINARY_0 && type <= MagnesiumValue.BINARY_127) { + return readBinary(type - MagnesiumValue.BINARY_0); + } else if (type > MagnesiumValue.BITS_0 && type < MagnesiumValue.BITS_1B) { + return readBits(type - MagnesiumValue.BITS_0); + } else if (type > MagnesiumValue.YIID_0) { + // Note 'byte' is range limited, so it is always '&& type <= MagnesiumValue.YIID_31' + return readYangInstanceIdentifier(type - MagnesiumValue.YIID_0); + } else { + throw new InvalidNormalizedNodeStreamException("Invalid value type " + type); + } + } + } + + abstract @NonNull BigInteger readBigInteger() throws IOException; + + private byte @NonNull [] readBinary(final int size) throws IOException { + if (size > 0) { + final byte[] ret = new byte[size]; + input.readFully(ret); + return ret; + } else if (size == 0) { + return BINARY_0; + } else { + throw new InvalidNormalizedNodeStreamException("Invalid binary length " + size); + } + } + + private @NonNull ImmutableSet readBits(final int size) throws IOException { + if (size > 0) { + final ImmutableSet.Builder builder = ImmutableSet.builder(); + for (int i = 0; i < size; ++i) { + builder.add(readRefString()); + } + return builder.build(); + } else if (size == 0) { + return ImmutableSet.of(); + } else { + throw new InvalidNormalizedNodeStreamException("Invalid bits length " + size); + } + } +} diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataOutput.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataOutput.java new file mode 100644 index 0000000000..c0dfb1bc5f --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/AbstractMagnesiumDataOutput.java @@ -0,0 +1,690 @@ +/* + * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.node.utils.stream; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.io.DataOutput; +import java.io.IOException; +import java.io.StringWriter; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import org.eclipse.jdt.annotation.NonNull; +import org.opendaylight.yangtools.rfc8528.data.api.MountPointIdentifier; +import org.opendaylight.yangtools.yang.common.Empty; +import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.common.QNameModule; +import org.opendaylight.yangtools.yang.common.Revision; +import org.opendaylight.yangtools.yang.common.Uint16; +import org.opendaylight.yangtools.yang.common.Uint32; +import org.opendaylight.yangtools.yang.common.Uint64; +import org.opendaylight.yangtools.yang.common.Uint8; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.AugmentationIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeWithValue; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract base class for NormalizedNodeDataOutput based on {@link MagnesiumNode}, {@link MagnesiumPathArgument} and + * {@link MagnesiumValue}. + */ +abstract class AbstractMagnesiumDataOutput extends AbstractNormalizedNodeDataOutput { + private static final Logger LOG = LoggerFactory.getLogger(AbstractMagnesiumDataOutput.class); + + // Marker for encoding state when we have entered startLeafNode() within a startMapEntry() and that leaf corresponds + // to a key carried within NodeIdentifierWithPredicates. + private static final Object KEY_LEAF_STATE = new Object(); + // Marker for nodes which have simple content and do not use END_NODE marker to terminate + private static final Object NO_ENDNODE_STATE = new Object(); + + private static final TransformerFactory TF = TransformerFactory.newInstance(); + + /** + * Stack tracking encoding state. In general we track the node identifier of the currently-open element, but there + * are a few other circumstances where we push other objects. See {@link #KEY_LEAF_STATE} and + * {@link #NO_ENDNODE_STATE}. + */ + private final Deque stack = new ArrayDeque<>(); + + // Coding maps + private final Map aidCodeMap = new HashMap<>(); + private final Map moduleCodeMap = new HashMap<>(); + private final Map stringCodeMap = new HashMap<>(); + private final Map qnameCodeMap = new HashMap<>(); + + AbstractMagnesiumDataOutput(final DataOutput output) { + super(output); + } + + @Override + public final void startLeafNode(final NodeIdentifier name) throws IOException { + final Object current = stack.peek(); + if (current instanceof NodeIdentifierWithPredicates) { + final QName qname = name.getNodeType(); + if (((NodeIdentifierWithPredicates) current).containsKey(qname)) { + writeQNameNode(MagnesiumNode.NODE_LEAF | MagnesiumNode.PREDICATE_ONE, qname); + stack.push(KEY_LEAF_STATE); + return; + } + } + + startSimpleNode(MagnesiumNode.NODE_LEAF, name); + } + + @Override + public final void startLeafSet(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_LEAFSET, name); + } + + @Override + public final void startOrderedLeafSet(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_LEAFSET_ORDERED, name); + } + + @Override + public final void startLeafSetEntryNode(final NodeWithValue name) throws IOException { + if (matchesParentQName(name.getNodeType())) { + output.writeByte(MagnesiumNode.NODE_LEAFSET_ENTRY); + stack.push(NO_ENDNODE_STATE); + } else { + startSimpleNode(MagnesiumNode.NODE_LEAFSET_ENTRY, name); + } + } + + @Override + public final void startContainerNode(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_CONTAINER, name); + } + + @Override + public final void startUnkeyedList(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_LIST, name); + } + + @Override + public final void startUnkeyedListItem(final NodeIdentifier name, final int childSizeHint) throws IOException { + startInheritedNode(MagnesiumNode.NODE_LIST_ENTRY, name); + } + + @Override + public final void startMapNode(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_MAP, name); + } + + @Override + public final void startMapEntryNode(final NodeIdentifierWithPredicates identifier, final int childSizeHint) + throws IOException { + final int size = identifier.size(); + if (size == 1) { + startInheritedNode((byte) (MagnesiumNode.NODE_MAP_ENTRY | MagnesiumNode.PREDICATE_ONE), identifier); + } else if (size == 0) { + startInheritedNode((byte) (MagnesiumNode.NODE_MAP_ENTRY | MagnesiumNode.PREDICATE_ZERO), identifier); + } else if (size < 256) { + startInheritedNode((byte) (MagnesiumNode.NODE_MAP_ENTRY | MagnesiumNode.PREDICATE_1B), identifier); + output.writeByte(size); + } else { + startInheritedNode((byte) (MagnesiumNode.NODE_MAP_ENTRY | MagnesiumNode.PREDICATE_4B), identifier); + output.writeInt(size); + } + + writePredicates(identifier); + } + + @Override + public final void startOrderedMapNode(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_MAP_ORDERED, name); + } + + @Override + public final void startChoiceNode(final NodeIdentifier name, final int childSizeHint) throws IOException { + startQNameNode(MagnesiumNode.NODE_CHOICE, name); + } + + @Override + public final void startAugmentationNode(final AugmentationIdentifier identifier) throws IOException { + final Integer code = aidCodeMap.get(identifier); + if (code == null) { + aidCodeMap.put(identifier, aidCodeMap.size()); + output.writeByte(MagnesiumNode.NODE_AUGMENTATION | MagnesiumNode.ADDR_DEFINE); + final Set qnames = identifier.getPossibleChildNames(); + output.writeInt(qnames.size()); + for (QName qname : qnames) { + writeQNameInternal(qname); + } + } else { + writeNodeType(MagnesiumNode.NODE_AUGMENTATION, code); + } + stack.push(identifier); + } + + @Override + public final void startAnyxmlNode(final NodeIdentifier name) throws IOException { + startSimpleNode(MagnesiumNode.NODE_ANYXML, name); + } + + @Override + public final void domSourceValue(final DOMSource value) throws IOException { + final StringWriter writer = new StringWriter(); + try { + TF.newTransformer().transform(value, new StreamResult(writer)); + } catch (TransformerException e) { + throw new IOException("Error writing anyXml", e); + } + writeValue(writer.toString()); + } + + @Override + public final void startYangModeledAnyXmlNode(final NodeIdentifier name, final int childSizeHint) + throws IOException { + // FIXME: implement this + throw new UnsupportedOperationException(); + } + + @Override + public final void endNode() throws IOException { + if (stack.pop() instanceof PathArgument) { + output.writeByte(MagnesiumNode.NODE_END); + } + } + + @Override + public final void scalarValue(final Object value) throws IOException { + if (KEY_LEAF_STATE.equals(stack.peek())) { + LOG.trace("Inside a map entry key leaf, not emitting value {}", value); + } else { + writeObject(value); + } + } + + @Override + final void writeQNameInternal(final QName qname) throws IOException { + final Integer code = qnameCodeMap.get(qname); + if (code == null) { + output.writeByte(MagnesiumValue.QNAME); + encodeQName(qname); + } else { + writeQNameRef(code); + } + } + + @Override + final void writePathArgumentInternal(final PathArgument pathArgument) throws IOException { + if (pathArgument instanceof NodeIdentifier) { + writeNodeIdentifier((NodeIdentifier) pathArgument); + } else if (pathArgument instanceof NodeIdentifierWithPredicates) { + writeNodeIdentifierWithPredicates((NodeIdentifierWithPredicates) pathArgument); + } else if (pathArgument instanceof AugmentationIdentifier) { + writeAugmentationIdentifier((AugmentationIdentifier) pathArgument); + } else if (pathArgument instanceof NodeWithValue) { + writeNodeWithValue((NodeWithValue) pathArgument); + } else if (pathArgument instanceof MountPointIdentifier) { + writeMountPointIdentifier((MountPointIdentifier) pathArgument); + } else { + throw new IOException("Unhandled PathArgument " + pathArgument); + } + } + + private void writeAugmentationIdentifier(final AugmentationIdentifier identifier) throws IOException { + final Set qnames = identifier.getPossibleChildNames(); + final int size = qnames.size(); + if (size < 29) { + output.writeByte(MagnesiumPathArgument.AUGMENTATION_IDENTIFIER + | size << MagnesiumPathArgument.AID_COUNT_SHIFT); + } else if (size < 256) { + output.writeByte(MagnesiumPathArgument.AUGMENTATION_IDENTIFIER | MagnesiumPathArgument.AID_COUNT_1B); + output.writeByte(size); + } else if (size < 65536) { + output.writeByte(MagnesiumPathArgument.AUGMENTATION_IDENTIFIER | MagnesiumPathArgument.AID_COUNT_2B); + output.writeShort(size); + } else { + output.writeByte(MagnesiumPathArgument.AUGMENTATION_IDENTIFIER | MagnesiumPathArgument.AID_COUNT_4B); + output.writeInt(size); + } + + for (QName qname : qnames) { + writeQNameInternal(qname); + } + } + + private void writeNodeIdentifier(final NodeIdentifier identifier) throws IOException { + writePathArgumentQName(identifier.getNodeType(), MagnesiumPathArgument.NODE_IDENTIFIER); + } + + private void writeMountPointIdentifier(final MountPointIdentifier identifier) throws IOException { + writePathArgumentQName(identifier.getNodeType(), MagnesiumPathArgument.MOUNTPOINT_IDENTIFIER); + } + + private void writeNodeIdentifierWithPredicates(final NodeIdentifierWithPredicates identifier) throws IOException { + final int size = identifier.size(); + if (size < 5) { + writePathArgumentQName(identifier.getNodeType(), + (byte) (MagnesiumPathArgument.NODE_IDENTIFIER_WITH_PREDICATES + | size << MagnesiumPathArgument.SIZE_SHIFT)); + } else if (size < 256) { + writePathArgumentQName(identifier.getNodeType(), + (byte) (MagnesiumPathArgument.NODE_IDENTIFIER_WITH_PREDICATES | MagnesiumPathArgument.SIZE_1B)); + output.writeByte(size); + } else if (size < 65536) { + writePathArgumentQName(identifier.getNodeType(), + (byte) (MagnesiumPathArgument.NODE_IDENTIFIER_WITH_PREDICATES | MagnesiumPathArgument.SIZE_2B)); + output.writeShort(size); + } else { + writePathArgumentQName(identifier.getNodeType(), + (byte) (MagnesiumPathArgument.NODE_IDENTIFIER_WITH_PREDICATES | MagnesiumPathArgument.SIZE_4B)); + output.writeInt(size); + } + + writePredicates(identifier); + } + + private void writePredicates(final NodeIdentifierWithPredicates identifier) throws IOException { + for (Entry e : identifier.entrySet()) { + writeQNameInternal(e.getKey()); + writeObject(e.getValue()); + } + } + + private void writeNodeWithValue(final NodeWithValue identifier) throws IOException { + writePathArgumentQName(identifier.getNodeType(), MagnesiumPathArgument.NODE_WITH_VALUE); + writeObject(identifier.getValue()); + } + + private void writePathArgumentQName(final QName qname, final byte typeHeader) throws IOException { + final Integer code = qnameCodeMap.get(qname); + if (code != null) { + final int val = code; + if (val < 256) { + output.writeByte(typeHeader | MagnesiumPathArgument.QNAME_REF_1B); + output.writeByte(val); + } else if (val < 65792) { + output.writeByte(typeHeader | MagnesiumPathArgument.QNAME_REF_2B); + output.writeShort(val - 256); + } else { + output.writeByte(typeHeader | MagnesiumPathArgument.QNAME_REF_4B); + output.writeInt(val); + } + } else { + // implied '| MagnesiumPathArgument.QNAME_DEF' + output.writeByte(typeHeader); + encodeQName(qname); + } + } + + @Override + final void writeYangInstanceIdentifierInternal(final YangInstanceIdentifier identifier) throws IOException { + writeValue(identifier); + } + + private void writeObject(final @NonNull Object value) throws IOException { + if (value instanceof String) { + writeValue((String) value); + } else if (value instanceof Boolean) { + writeValue((Boolean) value); + } else if (value instanceof Byte) { + writeValue((Byte) value); + } else if (value instanceof Short) { + writeValue((Short) value); + } else if (value instanceof Integer) { + writeValue((Integer) value); + } else if (value instanceof Long) { + writeValue((Long) value); + } else if (value instanceof Uint8) { + writeValue((Uint8) value); + } else if (value instanceof Uint16) { + writeValue((Uint16) value); + } else if (value instanceof Uint32) { + writeValue((Uint32) value); + } else if (value instanceof Uint64) { + writeValue((Uint64) value); + } else if (value instanceof QName) { + writeQNameInternal((QName) value); + } else if (value instanceof YangInstanceIdentifier) { + writeValue((YangInstanceIdentifier) value); + } else if (value instanceof byte[]) { + writeValue((byte[]) value); + } else if (value instanceof Empty) { + output.writeByte(MagnesiumValue.EMPTY); + } else if (value instanceof Set) { + writeValue((Set) value); + } else if (value instanceof BigDecimal) { + writeValue((BigDecimal) value); + } else if (value instanceof BigInteger) { + writeValue((BigInteger) value); + } else { + throw new IOException("Unhandled value type " + value.getClass()); + } + } + + private void writeValue(final boolean value) throws IOException { + output.writeByte(value ? MagnesiumValue.BOOLEAN_TRUE : MagnesiumValue.BOOLEAN_FALSE); + } + + private void writeValue(final byte value) throws IOException { + if (value != 0) { + output.writeByte(MagnesiumValue.INT8); + output.writeByte(value); + } else { + output.writeByte(MagnesiumValue.INT8_0); + } + } + + private void writeValue(final short value) throws IOException { + if (value != 0) { + output.writeByte(MagnesiumValue.INT16); + output.writeShort(value); + } else { + output.writeByte(MagnesiumValue.INT16_0); + } + } + + private void writeValue(final int value) throws IOException { + if ((value & 0xFFFF0000) != 0) { + output.writeByte(MagnesiumValue.INT32); + output.writeInt(value); + } else if (value != 0) { + output.writeByte(MagnesiumValue.INT32_2B); + output.writeShort(value); + } else { + output.writeByte(MagnesiumValue.INT32_0); + } + } + + private void writeValue(final long value) throws IOException { + if ((value & 0xFFFFFFFF00000000L) != 0) { + output.writeByte(MagnesiumValue.INT64); + output.writeLong(value); + } else if (value != 0) { + output.writeByte(MagnesiumValue.INT64_4B); + output.writeInt((int) value); + } else { + output.writeByte(MagnesiumValue.INT64_0); + } + } + + private void writeValue(final Uint8 value) throws IOException { + final byte b = value.byteValue(); + if (b != 0) { + output.writeByte(MagnesiumValue.UINT8); + output.writeByte(b); + } else { + output.writeByte(MagnesiumValue.UINT8_0); + } + } + + private void writeValue(final Uint16 value) throws IOException { + final short s = value.shortValue(); + if (s != 0) { + output.writeByte(MagnesiumValue.UINT16); + output.writeShort(s); + } else { + output.writeByte(MagnesiumValue.UINT16_0); + } + } + + private void writeValue(final Uint32 value) throws IOException { + final int i = value.intValue(); + if ((i & 0xFFFF0000) != 0) { + output.writeByte(MagnesiumValue.UINT32); + output.writeInt(i); + } else if (i != 0) { + output.writeByte(MagnesiumValue.UINT32_2B); + output.writeShort(i); + } else { + output.writeByte(MagnesiumValue.UINT32_0); + } + } + + private void writeValue(final Uint64 value) throws IOException { + final long l = value.longValue(); + if ((l & 0xFFFFFFFF00000000L) != 0) { + output.writeByte(MagnesiumValue.UINT64); + output.writeLong(l); + } else if (l != 0) { + output.writeByte(MagnesiumValue.UINT64_4B); + output.writeInt((int) l); + } else { + output.writeByte(MagnesiumValue.UINT64_0); + } + } + + private void writeValue(final BigDecimal value) throws IOException { + output.writeByte(MagnesiumValue.BIGDECIMAL); + output.writeUTF(value.toString()); + } + + abstract void writeValue(BigInteger value) throws IOException; + + private void writeValue(final String value) throws IOException { + if (value.isEmpty()) { + output.writeByte(MagnesiumValue.STRING_EMPTY); + } else if (value.length() <= Short.MAX_VALUE / 2) { + output.writeByte(MagnesiumValue.STRING_UTF); + output.writeUTF(value); + } else if (value.length() <= 1048576) { + final byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length < 65536) { + output.writeByte(MagnesiumValue.STRING_2B); + output.writeShort(bytes.length); + } else { + output.writeByte(MagnesiumValue.STRING_4B); + output.writeInt(bytes.length); + } + output.write(bytes); + } else { + output.writeByte(MagnesiumValue.STRING_CHARS); + output.writeInt(value.length()); + output.writeChars(value); + } + } + + private void writeValue(final byte[] value) throws IOException { + if (value.length < 128) { + output.writeByte(MagnesiumValue.BINARY_0 + value.length); + } else if (value.length < 384) { + output.writeByte(MagnesiumValue.BINARY_1B); + output.writeByte(value.length - 128); + } else if (value.length < 65920) { + output.writeByte(MagnesiumValue.BINARY_2B); + output.writeShort(value.length - 384); + } else { + output.writeByte(MagnesiumValue.BINARY_4B); + output.writeInt(value.length); + } + output.write(value); + } + + private void writeValue(final YangInstanceIdentifier value) throws IOException { + final List args = value.getPathArguments(); + final int size = args.size(); + if (size > 31) { + output.writeByte(MagnesiumValue.YIID); + output.writeInt(size); + } else { + output.writeByte(MagnesiumValue.YIID_0 + size); + } + for (PathArgument arg : args) { + writePathArgumentInternal(arg); + } + } + + private void writeValue(final Set value) throws IOException { + final int size = value.size(); + if (size < 29) { + output.writeByte(MagnesiumValue.BITS_0 + size); + } else if (size < 285) { + output.writeByte(MagnesiumValue.BITS_1B); + output.writeByte(size - 29); + } else if (size < 65821) { + output.writeByte(MagnesiumValue.BITS_2B); + output.writeShort(size - 285); + } else { + output.writeByte(MagnesiumValue.BITS_4B); + output.writeInt(size); + } + + for (Object bit : value) { + checkArgument(bit instanceof String, "Expected value type to be String but was %s", bit); + encodeString((String) bit); + } + } + + // Check if the proposed QName matches the parent. This is only effective if the parent is identified by + // NodeIdentifier -- which is typically true + private boolean matchesParentQName(final QName qname) { + final Object current = stack.peek(); + return current instanceof NodeIdentifier && qname.equals(((NodeIdentifier) current).getNodeType()); + } + + // Start an END_NODE-terminated node, which typically has a QName matching the parent. If that is the case we emit + // a parent reference instead of an explicit QName reference -- saving at least one byte + private void startInheritedNode(final byte type, final PathArgument name) throws IOException { + final QName qname = name.getNodeType(); + if (matchesParentQName(qname)) { + output.write(type); + } else { + writeQNameNode(type, qname); + } + stack.push(name); + } + + // Start an END_NODE-terminated node, which needs its QName encoded + private void startQNameNode(final byte type, final PathArgument name) throws IOException { + writeQNameNode(type, name.getNodeType()); + stack.push(name); + } + + // Start a simple node, which is not terminated through END_NODE and encode its QName + private void startSimpleNode(final byte type, final PathArgument name) throws IOException { + writeQNameNode(type, name.getNodeType()); + stack.push(NO_ENDNODE_STATE); + } + + // Encode a QName-based (i.e. NodeIdentifier*) node with a particular QName. This will either result in a QName + // definition, or a reference, where this is encoded along with the node type. + private void writeQNameNode(final int type, final @NonNull QName qname) throws IOException { + final Integer code = qnameCodeMap.get(qname); + if (code == null) { + output.writeByte(type | MagnesiumNode.ADDR_DEFINE); + encodeQName(qname); + } else { + writeNodeType(type, code); + } + } + + // Write a node type + lookup + private void writeNodeType(final int type, final int code) throws IOException { + if (code <= 255) { + output.writeByte(type | MagnesiumNode.ADDR_LOOKUP_1B); + output.writeByte(code); + } else { + output.writeByte(type | MagnesiumNode.ADDR_LOOKUP_4B); + output.writeInt(code); + } + } + + // Encode a QName using lookup tables, resuling either in a reference to an existing entry, or emitting two + // String values. + private void encodeQName(final @NonNull QName qname) throws IOException { + final Integer prev = qnameCodeMap.put(qname, qnameCodeMap.size()); + if (prev != null) { + throw new IOException("Internal coding error: attempted to re-encode " + qname + "%s already encoded as " + + prev); + } + + final QNameModule module = qname.getModule(); + final Integer code = moduleCodeMap.get(module); + if (code == null) { + moduleCodeMap.put(module, moduleCodeMap.size()); + encodeString(module.getNamespace().toString()); + final Optional rev = module.getRevision(); + if (rev.isPresent()) { + encodeString(rev.get().toString()); + } else { + output.writeByte(MagnesiumValue.STRING_EMPTY); + } + } else { + writeModuleRef(code); + } + encodeString(qname.getLocalName()); + } + + // Encode a String using lookup tables, resulting either in a reference to an existing entry, or emitting as + // a literal value + private void encodeString(final @NonNull String str) throws IOException { + final Integer code = stringCodeMap.get(str); + if (code != null) { + writeRef(code); + } else { + stringCodeMap.put(str, stringCodeMap.size()); + writeValue(str); + } + } + + // Write a QName with a lookup table reference. This is a combination of asserting the value is a QName plus + // the effects of writeRef() + private void writeQNameRef(final int code) throws IOException { + final int val = code; + if (val < 256) { + output.writeByte(MagnesiumValue.QNAME_REF_1B); + output.writeByte(val); + } else if (val < 65792) { + output.writeByte(MagnesiumValue.QNAME_REF_2B); + output.writeShort(val - 256); + } else { + output.writeByte(MagnesiumValue.QNAME_REF_4B); + output.writeInt(val); + } + } + + // Write a lookup table reference, which table is being referenced is implied by the caller + private void writeRef(final int code) throws IOException { + final int val = code; + if (val < 256) { + output.writeByte(MagnesiumValue.STRING_REF_1B); + output.writeByte(val); + } else if (val < 65792) { + output.writeByte(MagnesiumValue.STRING_REF_2B); + output.writeShort(val - 256); + } else { + output.writeByte(MagnesiumValue.STRING_REF_4B); + output.writeInt(val); + } + } + + // Write a lookup module table reference, which table is being referenced is implied by the caller + private void writeModuleRef(final int code) throws IOException { + final int val = code; + if (val < 256) { + output.writeByte(MagnesiumValue.MODREF_1B); + output.writeByte(val); + } else if (val < 65792) { + output.writeByte(MagnesiumValue.MODREF_2B); + output.writeShort(val - 256); + } else { + output.writeByte(MagnesiumValue.MODREF_4B); + output.writeInt(val); + } + } +} diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeInputOutput.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeInputOutput.java index 87aec1dc11..e134d09e7f 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeInputOutput.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeInputOutput.java @@ -67,6 +67,8 @@ public final class NormalizedNodeInputOutput { return new LithiumNormalizedNodeOutputStreamWriter(output); case NEON_SR2: return new NeonSR2NormalizedNodeOutputStreamWriter(output); + case SODIUM_SR1: + return new SodiumSR1DataOutput(output); default: throw new IllegalStateException("Unhandled version " + version); } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeStreamVersion.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeStreamVersion.java index a2b04c3255..9352347a22 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeStreamVersion.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/NormalizedNodeStreamVersion.java @@ -17,5 +17,6 @@ import org.eclipse.jdt.annotation.NonNullByDefault; @NonNullByDefault public enum NormalizedNodeStreamVersion { LITHIUM, - NEON_SR2; + NEON_SR2, + SODIUM_SR1; } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataInput.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataInput.java new file mode 100644 index 0000000000..d866411eae --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataInput.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.node.utils.stream; + +import java.io.DataInput; +import java.io.IOException; +import java.math.BigInteger; + +final class SodiumSR1DataInput extends AbstractMagnesiumDataInput { + SodiumSR1DataInput(final DataInput input) { + super(input); + } + + @Override + public NormalizedNodeStreamVersion getVersion() { + return NormalizedNodeStreamVersion.SODIUM_SR1; + } + + @Override + BigInteger readBigInteger() throws IOException { + // FIXME: use string -> BigInteger cache + return new BigInteger(input.readUTF()); + } +} diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataOutput.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataOutput.java new file mode 100644 index 0000000000..4c432d362a --- /dev/null +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/SodiumSR1DataOutput.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.datastore.node.utils.stream; + +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigInteger; + +final class SodiumSR1DataOutput extends AbstractMagnesiumDataOutput { + SodiumSR1DataOutput(final DataOutput output) { + super(output); + } + + @Override + short streamVersion() { + return TokenTypes.SODIUM_SR1_VERSION; + } + + @Override + void writeValue(final BigInteger value) throws IOException { + output.writeByte(MagnesiumValue.BIGINTEGER); + output.writeUTF(value.toString()); + } +} diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java index 43082affe1..e597cb95bd 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/TokenTypes.java @@ -23,4 +23,8 @@ final class TokenTypes { * dictionary, too. */ static final short NEON_SR2_VERSION = 2; + /** + * From-scratch designed version shipping in Sodium SR1. + */ + static final short SODIUM_SR1_VERSION = 3; } diff --git a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/VersionedNormalizedNodeDataInput.java b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/VersionedNormalizedNodeDataInput.java index e58d229eff..510b91dd15 100644 --- a/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/VersionedNormalizedNodeDataInput.java +++ b/opendaylight/md-sal/sal-clustering-commons/src/main/java/org/opendaylight/controller/cluster/datastore/node/utils/stream/VersionedNormalizedNodeDataInput.java @@ -40,6 +40,9 @@ final class VersionedNormalizedNodeDataInput extends ForwardingNormalizedNodeDat case TokenTypes.NEON_SR2_VERSION: ret = new NeonSR2NormalizedNodeInputStreamReader(input); break; + case TokenTypes.SODIUM_SR1_VERSION: + ret = new SodiumSR1DataInput(input); + break; default: throw defunct("Unhandled stream version %s", version); } -- 2.36.6