--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import static java.util.Objects.requireNonNull;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.misc.Pair;
+
+abstract class AbstractSourceToken extends AbstractToken {
+ private final Pair<TokenSource, CharStream> source;
+
+ AbstractSourceToken(final Pair<TokenSource, CharStream> source) {
+ this.source = requireNonNull(source);
+ }
+
+ @Override
+ public final TokenSource getTokenSource() {
+ return source.a;
+ }
+
+ @Override
+ public final CharStream getInputStream() {
+ return source.b;
+ }
+
+ @Override
+ public final String getText() {
+ final CharStream input = getInputStream();
+ if (input == null) {
+ return null;
+ }
+
+ final int n = input.size();
+ final int start = getStartIndex();
+ final int stop = getStopIndex();
+ return start < n && stop < n ? input.getText(Interval.of(start, stop)) : "<EOF>";
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import org.antlr.v4.runtime.WritableToken;
+
+abstract class AbstractToken implements WritableToken {
+ private int tokenIndex = -1;
+
+ @Override
+ public final int getChannel() {
+ return DEFAULT_CHANNEL;
+ }
+
+ @Override
+ public final int getTokenIndex() {
+ return tokenIndex;
+ }
+
+ @Override
+ public final void setTokenIndex(final int index) {
+ checkState(tokenIndex == -1);
+ tokenIndex = index;
+ }
+
+ @Override
+ public final void setText(final String text) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public final void setType(final int ttype) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public final void setLine(final int line) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public final void setCharPositionInLine(final int pos) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public final void setChannel(final int channel) {
+ throw new UnsupportedOperationException();
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CommonTokenFactory;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenFactory;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Pair;
+import org.eclipse.jdt.annotation.NonNull;
+
+/**
+ * A token factory which is more memory-efficient than {@link CommonTokenFactory}. We try to mimic behavior of common
+ * tokens, but do so at lower memory overheads, potentially sacrificing some performance.
+ */
+final class CompactTokenFactory implements TokenFactory<Token> {
+ static final @NonNull CompactTokenFactory INSTANCE = new CompactTokenFactory();
+
+ private CompactTokenFactory() {
+ // Hidden on purpose
+ }
+
+ @Override
+ public Token create(final Pair<TokenSource, CharStream> source, final int type, final String text,
+ final int channel, final int start, final int stop, final int line, final int charPositionInLine) {
+ if (channel != Token.DEFAULT_CHANNEL || text != null) {
+ // Non-default channel or text present, defer to common token factory
+ return CommonTokenFactory.DEFAULT.create(source, type, text, channel, start, stop, line,
+ charPositionInLine);
+ }
+
+ // Can we fit token type into a single byte? This should always be true
+ if (type >= Byte.MIN_VALUE && type <= Byte.MAX_VALUE) {
+ // Can we fit line in an unsigned short? This is usually be true
+ if (line >= 0 && line <= 65535) {
+ // Can we fit position in line into an unsigned byte? This is usually true
+ if (charPositionInLine >= 0 && charPositionInLine <= 255) {
+ // Can we fit start/stop into an an unsigned short?
+ if (start >= 0 && start <= 65535 && stop >= 0 && stop <= 65535) {
+ return new Token12122(source, type, line, charPositionInLine, start, stop);
+ }
+ return new Token12144(source, type, line, charPositionInLine, start, stop);
+ }
+ }
+ }
+
+ return new Token44444(source, type, line, charPositionInLine, start, stop);
+ }
+
+ @Override
+ public Token create(final int type, final String text) {
+ return new ExplicitTextToken(type, text);
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import com.google.common.annotations.Beta;
+import org.antlr.v4.runtime.CharStream;
+import org.opendaylight.yangtools.yang.parser.antlr.YangStatementLexer;
+
+/**
+ * A {@link YangStatementLexer} backed by more efficient token factory. Exact details are explicitly outside of
+ * specification.
+ */
+@Beta
+public class CompactYangStatementLexer extends YangStatementLexer {
+ public CompactYangStatementLexer(final CharStream input) {
+ super(input);
+ setTokenFactory(CompactTokenFactory.INSTANCE);
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import static java.util.Objects.requireNonNull;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.TokenSource;
+
+final class ExplicitTextToken extends AbstractToken {
+ private final int type;
+ private final String text;
+
+ ExplicitTextToken(final int type, final String text) {
+ this.type = type;
+ this.text = requireNonNull(text);
+ }
+
+ @Override
+ public String getText() {
+ return text;
+ }
+
+ @Override
+ public int getType() {
+ return type;
+ }
+
+ @Override
+ public int getLine() {
+ // TODO: this mimics CommonToken, but is probably not right
+ return 0;
+ }
+
+ @Override
+ public int getCharPositionInLine() {
+ return -1;
+ }
+
+ @Override
+ public int getStartIndex() {
+ // TODO: this mimics CommonToken, but is probably not right
+ return 0;
+ }
+
+ @Override
+ public int getStopIndex() {
+ // TODO: this mimics CommonToken, but is probably not right
+ return 0;
+ }
+
+ @Override
+ public TokenSource getTokenSource() {
+ return null;
+ }
+
+ @Override
+ public CharStream getInputStream() {
+ return null;
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Pair;
+
+/**
+ * Smallest general token implementation. This cuts {@code channel}, {@code text} fields completely, as we typically
+ * we do not use them. {@code type} and {@code charPositionInLine} are cut down to a single byte, while others are cut
+ * down to unsigned shorts.
+ *
+ * <p>
+ * This class ends up costing 24/40/32/32 bytes instead of 48/64/48/48 bytes, a saving of 33-50%, while being sufficient
+ * in most scenarios.
+ */
+final class Token12122 extends AbstractSourceToken {
+ private final byte type;
+ private final short line;
+ private final byte charPositionInLine;
+ private final short startIndex;
+ private final short stopIndex;
+
+ Token12122(final Pair<TokenSource, CharStream> source, final int type, final int line, final int charPositionInLine,
+ final int startIndex, final int stopIndex) {
+ super(source);
+ this.type = (byte) type;
+ this.line = (short) line;
+ this.charPositionInLine = (byte) charPositionInLine;
+ this.startIndex = (short) startIndex;
+ this.stopIndex = (short) stopIndex;
+ }
+
+ @Override
+ public int getType() {
+ return type;
+ }
+
+ @Override
+ public int getLine() {
+ return Short.toUnsignedInt(line);
+ }
+
+ @Override
+ public int getCharPositionInLine() {
+ return Byte.toUnsignedInt(charPositionInLine);
+ }
+
+ @Override
+ public int getStartIndex() {
+ return Short.toUnsignedInt(startIndex);
+ }
+
+ @Override
+ public int getStopIndex() {
+ return Short.toUnsignedInt(stopIndex);
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Pair;
+
+/**
+ * Intermediate general token implementation. This cuts {@code channel}, {@code text} fields completely, as we typically
+ * we do not use them. {@code type} and {@code charPositionInLine} are cut down to a single byte, and {@code line} is
+ * cut down to an unsigned short. {@code startIndex} and {@code stopIndex} are kept at full four-byte range, this making
+ * this implementation useful beyond 64K-char file mark.
+ *
+ * <p>
+ * This class ends up costing 32/48/32/32 bytes instead of 48/64/48/48 bytes, a saving of 33% in the same scenarios as
+ * {@link Token12122} across all possible file sizes.
+ */
+final class Token12144 extends AbstractSourceToken {
+ private final byte type;
+ private final short line;
+ private final byte charPositionInLine;
+
+ private final int startIndex;
+ private final int stopIndex;
+
+ Token12144(final Pair<TokenSource, CharStream> source, final int type, final int line, final int charPositionInLine,
+ final int startIndex, final int stopIndex) {
+ super(source);
+ this.type = (byte) type;
+ this.line = (short) line;
+ this.charPositionInLine = (byte) charPositionInLine;
+ this.startIndex = startIndex;
+ this.stopIndex = stopIndex;
+ }
+
+ @Override
+ public int getType() {
+ return type;
+ }
+
+ @Override
+ public int getLine() {
+ return Short.toUnsignedInt(line);
+ }
+
+ @Override
+ public int getCharPositionInLine() {
+ return Byte.toUnsignedInt(charPositionInLine);
+ }
+
+ @Override
+ public int getStartIndex() {
+ return startIndex;
+ }
+
+ @Override
+ public int getStopIndex() {
+ return stopIndex;
+ }
+}
--- /dev/null
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.parser.rfc7950.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Pair;
+
+/**
+ * Large general token implementation. This cuts {@code channel}, {@code text} fields completely, as we typically
+ * we do not use them. All other fields are retained..
+ *
+ * <p>
+ * This class ends up costing 40/56/40/48 bytes instead of 48/64/48/48 bytes, a saving of 12-33%, while still being
+ * applicable in all situations.
+ */
+final class Token44444 extends AbstractSourceToken {
+ private final int type;
+ private final int line;
+ private final int charPositionInLine;
+ private final int startIndex;
+ private final int stopIndex;
+
+ Token44444(final Pair<TokenSource, CharStream> source, final int type, final int line, final int charPositionInLine,
+ final int startIndex, final int stopIndex) {
+ super(source);
+ this.type = type;
+ this.line = line;
+ this.charPositionInLine = charPositionInLine;
+ this.startIndex = startIndex;
+ this.stopIndex = stopIndex;
+ }
+
+ @Override
+ public int getType() {
+ return type;
+ }
+
+ @Override
+ public int getLine() {
+ return line;
+ }
+
+ @Override
+ public int getCharPositionInLine() {
+ return charPositionInLine;
+ }
+
+ @Override
+ public int getStartIndex() {
+ return startIndex;
+ }
+
+ @Override
+ public int getStopIndex() {
+ return stopIndex;
+ }
+}
import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.FileContext;
import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.KeywordContext;
import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.StatementContext;
+import org.opendaylight.yangtools.yang.parser.rfc7950.antlr.CompactYangStatementLexer;
import org.opendaylight.yangtools.yang.parser.spi.source.PrefixToModule;
import org.opendaylight.yangtools.yang.parser.spi.source.QNameToStatementDefinition;
import org.opendaylight.yangtools.yang.parser.spi.source.SourceException;
private static StatementContext parseYangSource(final SourceIdentifier source, final InputStream stream)
throws IOException, YangSyntaxErrorException {
- final YangStatementLexer lexer = new YangStatementLexer(CharStreams.fromStream(stream));
+ final YangStatementLexer lexer = new CompactYangStatementLexer(CharStreams.fromStream(stream));
final YangStatementParser parser = new YangStatementParser(new CommonTokenStream(lexer));
// disconnect from console error output
lexer.removeErrorListeners();
Code Review / yangtools.git / commitdiff