From c4b572cd6d818d23e1d1d8d1883f388b43bb1160 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Fri, 14 Aug 2020 12:55:12 +0200 Subject: [PATCH] Add an explicit intermediate YANG representation ANTLR classes are geared towards parsing and raw speed, without much regards to memory usage. Most notably using ParseTree as our AST ends up retaining separators and various metadata which we do not really need. Add a simplistic intermediate representation for our AST, which where we pre-process ANTLR trees and de-duplicate strings. This leads to elimination of 90% of objects, with corresponding reduction of memory footprint. JIRA: YANGTOOLS-1130 Change-Id: Idf057bbb1e29280a8e378529401023fb87d18923 Signed-off-by: Robert Varga --- .../repo/SharedSchemaContextFactory.java | 9 +- .../parser/rfc7950/ir/AbstractIRObject.java | 20 ++ .../yang/parser/rfc7950/ir/IRArgument.java | 201 ++++++++++++++ .../yang/parser/rfc7950/ir/IRKeyword.java | 116 ++++++++ .../yang/parser/rfc7950/ir/IRStatement.java | 109 ++++++++ .../parser/rfc7950/ir/IRStatement022.java | 29 ++ .../parser/rfc7950/ir/IRStatement031.java | 27 ++ .../parser/rfc7950/ir/IRStatement044.java | 29 ++ .../parser/rfc7950/ir/IRStatement144.java | 28 ++ .../parser/rfc7950/ir/IRStatementL44.java | 28 ++ .../parser/rfc7950/ir/StatementFactory.java | 248 ++++++++++++++++++ .../yang/parser/rfc7950/ir/package-info.java | 18 ++ .../parser/rfc7950/repo/ASTSchemaSource.java | 22 +- .../rfc7950/repo/ArgumentContextUtils.java | 188 ++----------- .../rfc7950/repo/StatementContextVisitor.java | 86 +++--- .../rfc7950/repo/TextToASTTransformer.java | 3 +- .../rfc7950/repo/YangModelDependencyInfo.java | 121 ++++----- .../repo/YangStatementStreamSource.java | 84 ++---- .../rfc7950/ir/StatementFactoryTest.java | 39 +++ .../repo/ArgumentContextUtilsTest.java | 25 -- 20 files changed, 1044 insertions(+), 386 deletions(-) create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/AbstractIRObject.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRArgument.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRKeyword.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement022.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement031.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement044.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement144.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatementL44.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/StatementFactory.java create mode 100644 yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/package-info.java create mode 100644 yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/StatementFactoryTest.java diff --git a/yang/yang-parser-impl/src/main/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaContextFactory.java b/yang/yang-parser-impl/src/main/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaContextFactory.java index b5ee5fd53e..87ab72d555 100644 --- a/yang/yang-parser-impl/src/main/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaContextFactory.java +++ b/yang/yang-parser-impl/src/main/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaContextFactory.java @@ -7,7 +7,6 @@ */ package org.opendaylight.yangtools.yang.parser.repo; -import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; import static org.opendaylight.yangtools.util.concurrent.FluentFutures.immediateFluentFuture; @@ -34,7 +33,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ExecutionException; -import org.antlr.v4.runtime.ParserRuleContext; import org.eclipse.jdt.annotation.NonNull; import org.gaul.modernizer_maven_annotations.SuppressModernizer; import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext; @@ -47,7 +45,6 @@ import org.opendaylight.yangtools.yang.model.repo.api.SchemaContextFactoryConfig import org.opendaylight.yangtools.yang.model.repo.api.SchemaResolutionException; import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier; import org.opendaylight.yangtools.yang.model.repo.api.StatementParserMode; -import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.StatementContext; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.ASTSchemaSource; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.YangModelDependencyInfo; import org.opendaylight.yangtools.yang.parser.spi.meta.ReactorException; @@ -231,12 +228,8 @@ final class SharedSchemaContextFactory implements EffectiveModelContextFactory { for (final Entry entry : srcs.entrySet()) { final ASTSchemaSource ast = entry.getValue(); - final ParserRuleContext parserRuleCtx = ast.getAST(); - checkArgument(parserRuleCtx instanceof StatementContext, "Unsupported context class %s for source %s", - parserRuleCtx.getClass(), entry.getKey()); - try { - parser.addSource(entry.getValue()); + parser.addSource(ast); } catch (YangSyntaxErrorException | IOException e) { throw new SchemaResolutionException("Failed to add source " + entry.getKey(), e); } diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/AbstractIRObject.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/AbstractIRObject.java new file mode 100644 index 0000000000..ab7102869f --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/AbstractIRObject.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import com.google.common.base.MoreObjects; +import org.opendaylight.yangtools.concepts.Immutable; + +abstract class AbstractIRObject implements Immutable { + @Override + public final String toString() { + return MoreObjects.toStringHelper(this).add("fragment", toYangFragment(new StringBuilder())).toString(); + } + + abstract StringBuilder toYangFragment(StringBuilder sb); +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRArgument.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRArgument.java new file mode 100644 index 0000000000..87aec4e221 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRArgument.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import static java.util.Objects.requireNonNull; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; +import java.util.Iterator; +import java.util.List; +import org.eclipse.jdt.annotation.NonNull; + +/** + * An argument to a YANG statement, as defined by section 6.1.3 of both + * RFC6020 and + * RFC7950. An argument is effectively any old string, + * except it can be defined in a number of ways: + *
    + *
  • it can be a simple unquoted string, or
    • + *
  • it can be a single-quoted string, with its contents being completely preserved, or
    • + *
  • it can be a double-quoted string, which defines some escaping and whitespace-stripping rules, or
    • + *
  • it can be a concatenation of any number of single- or double-quoted strings
    • + *
+ * + *

+ * The first three cases as covered by {@link Single} subclass, which exposes appropriate methods to infer how its + * string literal is to be interpreted. The last case is handled by {@link Concatenation} subclass, which exposes + * the constituent parts as {@link Single} items. + * + *

+ * Please note that parser implementations producing these argument representations are NOT required to retain + * the format of the original definition. They are free to perform quoting and concatenation transformations as long as + * they maintain semantic equivalence. As a matter of example, these transformations are explicitly allowed: + *

    + *
  • elimination of unneeded quotes, for example turning {@code "foo"} into {@code foo}
    • + *
  • transformation of quotes, for example turning {@code "foo\nbar"} into {@code 'foo bar'}
    • + *
  • concatenation processing, for example turning {@code 'foo' + 'bar'} into {@code foobar}
    • + *
+ */ +@Beta +public abstract class IRArgument extends AbstractIRObject { + /** + * An argument composed of multiple concatenated parts. + */ + public static final class Concatenation extends IRArgument { + private final @NonNull ImmutableList parts; + + Concatenation(final ImmutableList parts) { + this.parts = requireNonNull(parts); + } + + /** + * Return the argument parts that need to be concatenated. + * + * @return Argument parts. + */ + public @NonNull List parts() { + return parts; + } + + @Override + StringBuilder toYangFragment(final StringBuilder sb) { + final Iterator it = parts.iterator(); + it.next().toYangFragment(sb); + while (it.hasNext()) { + it.next().toYangFragment(sb.append(" + ")); + } + return sb; + } + } + + /** + * An argument composed of a single string. This string may need further validation and processing, as it may not + * actually conform to the specification as requested by {@code yang-version}. + */ + /* + * This is the public footprint which is served by three final subclasses: DoublyQuoted, SingleQuoted, Unquoted. + * Those classes must never be exposed, as they are a manifestation of current implementation in StatementFactory. + * As noted in the interface contract of IRArgument, we have very much free reign on syntactic transformations, + * StatementFactory is just not taking advantage of those at this point. + * + * The subclasses may very much change, in terms of both naming and function, to support whatever StatementFactory + * ends up doing. + */ + public abstract static class Single extends IRArgument { + private final @NonNull String string; + + Single(final String string) { + this.string = requireNonNull(string); + } + + /** + * Significant portion of this argument. For unquoted and single-quoted strings this is the unquoted string + * literal. For double-quoted strings this is the unquoted string, after whitespace trimming as defined by + * RFC6020/RFC7950 section 6.1.3, but before escape substitution. + * + * @return Significant portion of this argument. + */ + public final @NonNull String string() { + return string; + } + + /** + * Imprecise check if this argument needs further unescape operation (which is version-specific) to arrive at + * the literal string value. This is false for unquoted and single-quoted strings, which do not support any sort + * of escaping. This may be true for double-quoted strings, as they may need to be further processed in + * version-dependent ways to arrive at the correct literal value. + * + *

+ * This method is allowed to err on the false-positive side -- i.e. it may report any double-quoted string as + * needing further processing, even when the actual content could be determined to not need further processing. + * + * @return False if the value of {@link #string} can be used as-is. + */ + public final boolean needUnescape() { + return this instanceof DoubleQuoted; + } + + /** + * Imprecise check if this argument needs an additional content check for compliance. This is false if the + * string was explicitly quoted and therefore cannot contain stray single- or double-quotes, or if the content + * has already been checked to not contain them. + * + *

+ * The content check is needed to ascertain RFC7950 compliance, because RFC6020 allows constructs like + * 

abc"def
in unquoted strings, while RFC7950 explicitly forbids them. + * + *

+ * This method is allowed to err on the false-positive side -- i.e. it may report any unquoted string as + * needing this check, even when the actual content could be determined to not contain quotes. + * + * @return True if this argument requires a version-specific check for quote content. + */ + public final boolean needQuoteCheck() { + return this instanceof Unquoted; + } + + /** + * Imprecise check if this argument complies with the {@code identifier} YANG specification. + * + *

+ * This method is allowed to err on the false-negative side -- i.e. it may report any string as not being + * compliant with {@code identifier}, even when the actual content could be determined to be compliant. + * + * @return True if this argument is known to be directly usable in contexts where YANG requires the use of + */ + public final boolean isValidIdentifier() { + return this instanceof Identifier; + } + + @Override + StringBuilder toYangFragment(final StringBuilder sb) { + return sb.append(string); + } + } + + static final class DoubleQuoted extends Single { + DoubleQuoted(final String string) { + super(string); + } + + @Override + StringBuilder toYangFragment(final StringBuilder sb) { + // Note this is just an approximation. We do not have enough state knowledge to restore any whitespace we + // may have trimmed. + return super.toYangFragment(sb.append('"')).append('"'); + } + } + + static final class SingleQuoted extends Single { + SingleQuoted(final String string) { + super(string); + } + + @Override + StringBuilder toYangFragment(final StringBuilder sb) { + return super.toYangFragment(sb.append('\'')).append('\''); + } + } + + static final class Identifier extends Single { + Identifier(final String string) { + super(string); + } + } + + static final class Unquoted extends Single { + Unquoted(final String string) { + super(string); + } + } + + IRArgument() { + // Hidden on purpose + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRKeyword.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRKeyword.java new file mode 100644 index 0000000000..0df3c4d2e9 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRKeyword.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import static java.util.Objects.requireNonNull; + +import com.google.common.annotations.Beta; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; +import org.opendaylight.yangtools.yang.common.AbstractQName; + +/** + * A YANG keyword, as defined, as defined by section 6.1.2 of both + * RFC6020 and + * RFC7950. The two options are discerned by nullability + * of {@link #prefix()} method's return, as hinted by the ABNF for {@code node-identifier} -- and while a keyword is a + * semantically different construct, it shares the same value space. + * + *

+ * Naming in this class prefers the formal ABNF specification and draws value-space and type-safety implications from + * that connection, rather than following the RFC-assigned names. + */ +@Beta +public abstract class IRKeyword extends AbstractIRObject { + @Beta + public static final class Qualified extends IRKeyword { + private final @NonNull String prefix; + + Qualified(final String prefix, final String localName) { + super(localName); + this.prefix = requireNonNull(prefix); + } + + @Override + public @NonNull String prefix() { + return prefix; + } + + @Override + public String asStringDeclaration() { + return prefix + ':' + identifier(); + } + + @Override + StringBuilder toYangFragment(final StringBuilder sb) { + return sb.append(prefix).append(':').append(identifier()); + } + } + + @Beta + public static final class Unqualified extends IRKeyword { + Unqualified(final String localName) { + super(localName); + } + + @Override + public String prefix() { + return null; + } + + @Override + public String asStringDeclaration() { + return identifier(); + } + + @Override + StringBuilder toYangFragment(final StringBuilder sb) { + return sb.append(identifier()); + } + } + + private final @NonNull String identifier; + + IRKeyword(final String localName) { + this.identifier = requireNonNull(localName); + } + + /** + * This keyword's 'identifier' part. This corresponds to what the RFCs refer to as {@code YANG keyword} or as + * {@code language extension keyword}. + * + *

+ * Note the returned string is guaranteed to conform to rules of {@code identifier} ABNF and therefore + * is directly usable as a {@code localName} in an {@link AbstractQName}. + * + * @return This keyword's identifier part. + */ + public final @NonNull String identifier() { + return identifier; + } + + /** + * This keyword's 'prefix' part. This corresponds to {@code prefix identifier}. For {@code YANG keyword}s this is + * null. For language extension references this is the non-null prefix which references the YANG module defining + * the language extension. + * + *

+ * Note the returned string, if non-null, is guaranteed to conform to rules of {@code identifier} ABNF and therefore + * is directly usable as a {@code localName} in an {@link AbstractQName}. + * + * @return This keyword's prefix, or null if this keyword references a YANG keyword. + */ + public abstract @Nullable String prefix(); + + /** + * Helper method to re-create the string which was used to declared this keyword. + * + * @return Declaration string. + */ + public abstract @NonNull String asStringDeclaration(); +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement.java new file mode 100644 index 0000000000..adaf5dbf2c --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import static java.util.Objects.requireNonNull; + +import com.google.common.annotations.Beta; +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.StatementContext; + +/** + * A single YANG statement in its raw string form. A statement is composed of: + *

    + *
  • a mandatory keyword, modeled as {@link IRKeyword}
    • + *
  • an optional argument, modeled as {@link IRArgument}
    • + *
  • zero or more nested statements
    • + *
+ */ +@Beta +public abstract class IRStatement extends AbstractIRObject { + private final @NonNull IRKeyword keyword; + private final IRArgument argument; + + IRStatement(final IRKeyword keyword, final IRArgument argument) { + this.keyword = requireNonNull(keyword); + this.argument = argument; + } + + /** + * Create an {@link IRStatement} from a parsed {@link StatementContext}. + * + * @param context ANTLR statement context + * @return A new IRStatement + * @throws NullPointerException if {@code context} is null + */ + public static @NonNull IRStatement forContext(final StatementContext context) { + return new StatementFactory().createStatement(context); + } + + /** + * Return this statement's keyword. + * + * @return This statement's keyword. + */ + public final @NonNull IRKeyword keyword() { + return keyword; + } + + /** + * Return this statement's argument, if it is present. + * + * @return This statement's argument, or null if this statement does not have an argument + */ + public final @Nullable IRArgument argument() { + return argument; + } + + /** + * Return this statement's substatements. + * + * @return This statement's substatements. + */ + public @NonNull List statements() { + return ImmutableList.of(); + } + + /** + * Return the line number on which this statement's keyword has its first character, counting from 1. This + * information is used only for diagnostic purposes. + * + * @return Line number where this statement started in the source code. + */ + public abstract int startLine(); + + /** + * Return the column number on which this statement's keyword has its first character, counting from 0. This + * information is used only for diagnostic purposes. + * + * @return Column number where this statement started in the source code. + */ + public abstract int startColumn(); + + @Override + final StringBuilder toYangFragment(final StringBuilder sb) { + sb.append(keyword); + if (argument != null) { + argument.toYangFragment(sb.append(' ')); + } + + final List statements = statements(); + if (statements.isEmpty()) { + return sb.append(';'); + } + + sb.append(" {\n"); + for (IRStatement stmt : statements) { + stmt.toYangFragment(sb).append('\n'); + } + return sb.append('}'); + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement022.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement022.java new file mode 100644 index 0000000000..0d63bcb8e3 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement022.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +final class IRStatement022 extends IRStatement { + private final short startLine; + private final short startColumn; + + IRStatement022(final IRKeyword keyword, final IRArgument argument, final int startLine, final int startColumn) { + super(keyword, argument); + this.startLine = (short) startLine; + this.startColumn = (short) startColumn; + } + + @Override + public int startLine() { + return Short.toUnsignedInt(startLine); + } + + @Override + public int startColumn() { + return Short.toUnsignedInt(startColumn); + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement031.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement031.java new file mode 100644 index 0000000000..edd424b4cb --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement031.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +final class IRStatement031 extends IRStatement { + private final int value; + + IRStatement031(final IRKeyword keyword, final IRArgument argument, final int startLine, final int startColumn) { + super(keyword, argument); + this.value = startLine << 8 | startColumn & 0xFF; + } + + @Override + public int startLine() { + return value >>> 8; + } + + @Override + public int startColumn() { + return value & 0xFF; + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement044.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement044.java new file mode 100644 index 0000000000..ed8dfc69fa --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement044.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +class IRStatement044 extends IRStatement { + private final int startLine; + private final int startColumn; + + IRStatement044(final IRKeyword keyword, final IRArgument argument, final int startLine, final int startColumn) { + super(keyword, argument); + this.startLine = startLine; + this.startColumn = startColumn; + } + + @Override + public final int startLine() { + return startLine; + } + + @Override + public final int startColumn() { + return startColumn; + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement144.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement144.java new file mode 100644 index 0000000000..2a7e894a05 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatement144.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import static java.util.Objects.requireNonNull; + +import com.google.common.collect.ImmutableList; +import org.eclipse.jdt.annotation.NonNull; + +final class IRStatement144 extends IRStatement044 { + private final @NonNull IRStatement statement; + + IRStatement144(final IRKeyword keyword, final IRArgument argument, final IRStatement statement, + final int startLine, final int startColumn) { + super(keyword, argument, startLine, startColumn); + this.statement = requireNonNull(statement); + } + + @Override + public ImmutableList statements() { + return ImmutableList.of(statement); + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatementL44.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatementL44.java new file mode 100644 index 0000000000..c6acf69045 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/IRStatementL44.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import static java.util.Objects.requireNonNull; + +import com.google.common.collect.ImmutableList; +import org.eclipse.jdt.annotation.NonNull; + +final class IRStatementL44 extends IRStatement044 { + private final @NonNull ImmutableList statements; + + IRStatementL44(final IRKeyword keyword, final IRArgument argument, final ImmutableList statements, + final int startLine, final int startColumn) { + super(keyword, argument, startLine, startColumn); + this.statements = requireNonNull(statements); + } + + @Override + public ImmutableList statements() { + return statements; + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/StatementFactory.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/StatementFactory.java new file mode 100644 index 0000000000..c2e445b6b4 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/StatementFactory.java @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; + +import static com.google.common.base.Verify.verify; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.base.VerifyException; +import com.google.common.collect.ImmutableList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Function; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.eclipse.jdt.annotation.NonNull; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.ArgumentContext; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.KeywordContext; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.QuotedStringContext; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.StatementContext; +import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.UnquotedStringContext; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Concatenation; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.DoubleQuoted; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Identifier; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Single; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.SingleQuoted; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Unquoted; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRKeyword.Qualified; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRKeyword.Unqualified; + +final class StatementFactory { + private static final CharMatcher WHITESPACE_MATCHER = CharMatcher.whitespace(); + + private final Map dquotArguments = new HashMap<>(); + private final Map squotArguments = new HashMap<>(); + private final Map uquotArguments = new HashMap<>(); + private final Map idenArguments = new HashMap<>(); + private final Map uqualKeywords = new HashMap<>(); + private final Map, Qualified> qualKeywords = new HashMap<>(); + private final Map strings = new HashMap<>(); + + @NonNull IRStatement createStatement(final StatementContext stmt) { + final ParseTree firstChild = stmt.getChild(0); + verify(firstChild instanceof KeywordContext, "Unexpected shape of %s", stmt); + + final ParseTree keywordStart = firstChild.getChild(0); + verify(keywordStart instanceof TerminalNode, "Unexpected keyword start %s", keywordStart); + final Token keywordToken = ((TerminalNode) keywordStart).getSymbol(); + + final IRKeyword keyword; + switch (firstChild.getChildCount()) { + case 1: + keyword = uqualKeywords.computeIfAbsent(strOf(keywordToken), Unqualified::new); + break; + case 3: + keyword = qualKeywords.computeIfAbsent(Map.entry(strOf(keywordToken), strOf(firstChild.getChild(2))), + entry -> new Qualified(entry.getKey(), entry.getValue())); + break; + default: + throw new VerifyException("Unexpected keyword " + firstChild); + } + + final IRArgument argument = createArgument(stmt); + final ImmutableList statements = createStatements(stmt); + final int line = keywordToken.getLine(); + final int column = keywordToken.getCharPositionInLine(); + + switch (statements.size()) { + case 0: + return createStatement(keyword, argument, line, column); + case 1: + return new IRStatement144(keyword, argument, statements.get(0), line, column); + default: + return new IRStatementL44(keyword, argument, statements, line, column); + } + } + + private static @NonNull IRStatement createStatement(final IRKeyword keyword, final IRArgument argument, + final int line, final int column) { + if (line >= 0 && column >= 0) { + if (line <= 65535 && column <= 65535) { + return new IRStatement022(keyword, argument, line, column); + } + if (line <= 16777215 && column <= 255) { + return new IRStatement031(keyword, argument, line, column); + } + } + return new IRStatement044(keyword, argument, line, column); + } + + private IRArgument createArgument(final StatementContext stmt) { + final ArgumentContext argCtx = stmt.argument(); + if (argCtx == null) { + return null; + } + if (argCtx.getChildCount() == 1) { + final ParseTree child = argCtx.getChild(0); + if (child instanceof TerminalNode) { + // This is as simple as it gets: we are dealing with an identifier here. + return idenArguments.computeIfAbsent(strOf(((TerminalNode) child).getSymbol()), Identifier::new); + } + if (child instanceof UnquotedStringContext) { + // TODO: check non-presence of quotes and create a different subclass, so that ends up treated as if it + // was single-quoted, i.e. bypass the check implied by IRArgument.Single#needQuoteCheck(). + return uquotArguments.computeIfAbsent(strOf(child), Unquoted::new); + } + + verify(child instanceof QuotedStringContext, "Unexpected child %s", child); + return createArgument((QuotedStringContext) child); + } + + // TODO: perform concatenation of single-quoted strings. For double-quoted strings this may not be as nice, but + // for single-quoted strings we do not need further validation in in the reactor and can use them as raw + // literals. This saves some indirection overhead (on memory side) and can slightly improve execution + // speed when we process the same IR multiple times. + + return new Concatenation(argCtx.quotedString().stream().map(this::createArgument) + .collect(ImmutableList.toImmutableList())); + } + + private Single createArgument(final QuotedStringContext argument) { + final ParseTree literal = argument.getChild(1); + verify(literal instanceof TerminalNode, "Unexpected literal %s", literal); + final Token token = ((TerminalNode) literal).getSymbol(); + switch (token.getType()) { + case YangStatementParser.DQUOT_END: + return dquotArguments.computeIfAbsent("", DoubleQuoted::new); + case YangStatementParser.DQUOT_STRING: + // Whitespace normalization happens irrespective of further handling and has no effect on the result + final String str = intern(trimWhitespace(token.getText(), token.getCharPositionInLine() - 1)); + + // TODO: turn this into a single-quoted literal if a backslash is not present. Doing so allows the + // argument to be treated as a literal. See IRArgument.Single#needUnescape() for more context. + // This may look unimportant, but there are scenarios where we process the same AST multiple times + // and remembering this detail saves a string scan. + + return dquotArguments.computeIfAbsent(str, DoubleQuoted::new); + case YangStatementParser.SQUOT_END: + return squotArguments.computeIfAbsent("", SingleQuoted::new); + case YangStatementParser.SQUOT_STRING: + return squotArguments.computeIfAbsent(strOf(token), SingleQuoted::new); + default: + throw new VerifyException("Unexpected token " + token); + } + } + + private ImmutableList createStatements(final StatementContext stmt) { + final List statements = stmt.statement(); + return statements.isEmpty() ? ImmutableList.of() + : statements.stream().map(this::createStatement).collect(ImmutableList.toImmutableList()); + } + + private String strOf(final ParseTree tree) { + return intern(tree.getText()); + } + + private String strOf(final Token token) { + return intern(token.getText()); + } + + private String intern(final String str) { + return strings.computeIfAbsent(str, Function.identity()); + } + + @VisibleForTesting + static String trimWhitespace(final String str, final int dquot) { + final int firstBrk = str.indexOf('\n'); + if (firstBrk == -1) { + return str; + } + + // Okay, we may need to do some trimming, set up a builder and append the first segment + final int length = str.length(); + final StringBuilder sb = new StringBuilder(length); + + // Append first segment, which needs only tail-trimming + sb.append(str, 0, trimTrailing(str, 0, firstBrk)).append('\n'); + + // With that out of the way, setup our iteration state. The string segment we are looking at is + // str.substring(start, end), which is guaranteed not to include any line breaks, i.e. end <= brk unless we are + // at the last segment. + int start = firstBrk + 1; + int brk = str.indexOf('\n', start); + + // Loop over inner strings + while (brk != -1) { + trimLeadingAndAppend(sb, dquot, str, start, trimTrailing(str, start, brk)).append('\n'); + start = brk + 1; + brk = str.indexOf('\n', start); + } + + return trimLeadingAndAppend(sb, dquot, str, start, length).toString(); + } + + private static StringBuilder trimLeadingAndAppend(final StringBuilder sb, final int dquot, final String str, + final int start, final int end) { + int offset = start; + int pos = 0; + + while (pos <= dquot) { + if (offset == end) { + // We ran out of data, nothing to append + return sb; + } + + final char ch = str.charAt(offset); + if (ch == '\t') { + // tabs are to be treated as 8 spaces + pos += 8; + } else if (WHITESPACE_MATCHER.matches(ch)) { + pos++; + } else { + break; + } + + offset++; + } + + // We have expanded beyond double quotes, push equivalent spaces + while (pos - 1 > dquot) { + sb.append(' '); + pos--; + } + + return sb.append(str, offset, end); + } + + private static int trimTrailing(final String str, final int start, final int end) { + int ret = end; + while (ret > start) { + final int prev = ret - 1; + if (!WHITESPACE_MATCHER.matches(str.charAt(prev))) { + break; + } + ret = prev; + } + return ret; + } +} diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/package-info.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/package-info.java new file mode 100644 index 0000000000..5b14723023 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/ir/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +/** + * Intermediate representation of a YANG file. This is an Abstract Syntax Tree equivalent to ParseTree we get from + * ANTLR, except it is immutable and has a denser in-memory representation due to it not containing any metadata which + * is not required for the purposes of statement inference. + * + *

+ * The main entrypoint into this package is {@link org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRStatement}, which + * represents a single YANG statement. Every YANG file is required to contain exactly one top-level statement, + * {@code module} or {@code submodule}, hence an IRStatement also represents the significant contents of a YANG file. + */ +package org.opendaylight.yangtools.yang.parser.rfc7950.ir; \ No newline at end of file diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ASTSchemaSource.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ASTSchemaSource.java index cf059e59e4..9cf3717a76 100644 --- a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ASTSchemaSource.java +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ASTSchemaSource.java @@ -11,7 +11,6 @@ import static java.util.Objects.requireNonNull; import com.google.common.annotations.Beta; import java.util.Optional; -import org.antlr.v4.runtime.ParserRuleContext; import org.eclipse.jdt.annotation.NonNull; import org.eclipse.jdt.annotation.Nullable; import org.opendaylight.yangtools.concepts.AbstractIdentifiable; @@ -20,6 +19,7 @@ import org.opendaylight.yangtools.yang.model.repo.api.RevisionSourceIdentifier; import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceRepresentation; import org.opendaylight.yangtools.yang.model.repo.api.SemVerSourceIdentifier; import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier; +import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRStatement; /** * Abstract Syntax Tree representation of a schema source. This representation is internal to the YANG parser @@ -35,15 +35,15 @@ public final class ASTSchemaSource extends AbstractIdentifiable