2 * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.yangtools.yang.parser.rfc7950.ir;
10 import static com.google.common.base.Verify.verify;
12 import com.google.common.annotations.Beta;
13 import com.google.common.annotations.VisibleForTesting;
14 import com.google.common.base.CharMatcher;
15 import com.google.common.base.VerifyException;
16 import com.google.common.collect.ImmutableList;
17 import java.util.ArrayList;
18 import java.util.HashMap;
19 import java.util.List;
21 import java.util.Map.Entry;
22 import java.util.function.Function;
23 import org.antlr.v4.runtime.Token;
24 import org.antlr.v4.runtime.tree.ParseTree;
25 import org.antlr.v4.runtime.tree.TerminalNode;
26 import org.eclipse.jdt.annotation.NonNull;
27 import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser;
28 import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.ArgumentContext;
29 import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.FileContext;
30 import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.KeywordContext;
31 import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.StatementContext;
32 import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.UnquotedStringContext;
33 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Concatenation;
34 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.DoubleQuoted;
35 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Identifier;
36 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Single;
37 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.SingleQuoted;
38 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRArgument.Unquoted;
39 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRKeyword.Qualified;
40 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRKeyword.Unqualified;
43 public final class AntlrSupport {
44 private static final CharMatcher WHITESPACE_MATCHER = CharMatcher.whitespace();
46 private final Map<String, DoubleQuoted> dquotArguments = new HashMap<>();
47 private final Map<String, SingleQuoted> squotArguments = new HashMap<>();
48 private final Map<String, Unquoted> uquotArguments = new HashMap<>();
49 private final Map<String, Identifier> idenArguments = new HashMap<>();
50 private final Map<String, Unqualified> uqualKeywords = new HashMap<>();
51 private final Map<Entry<String, String>, Qualified> qualKeywords = new HashMap<>();
52 private final Map<String, String> strings = new HashMap<>();
54 private AntlrSupport() {
59 * Create an {@link IRStatement} from a parsed {@link FileContext}.
61 * @param file ANTLR file context
62 * @return A new IRStatement
63 * @throws NullPointerException if {@code file} is null or it does not contain a root statement
65 public static @NonNull IRStatement createStatement(@SuppressWarnings("exports") final FileContext file) {
66 return createStatement(file.statement());
70 * Create an {@link IRStatement} from a parsed {@link StatementContext}.
72 * @param stmt ANTLR statement context
73 * @return A new IRStatement
74 * @throws NullPointerException if {@code stmt} is null
76 public static @NonNull IRStatement createStatement(@SuppressWarnings("exports") final StatementContext stmt) {
77 return new AntlrSupport().statementOf(stmt);
80 private @NonNull IRStatement statementOf(final StatementContext stmt) {
81 final ParseTree firstChild = stmt.getChild(0);
82 verify(firstChild instanceof KeywordContext, "Unexpected shape of %s", stmt);
84 final ParseTree keywordStart = firstChild.getChild(0);
85 verify(keywordStart instanceof TerminalNode, "Unexpected keyword start %s", keywordStart);
86 final Token keywordToken = ((TerminalNode) keywordStart).getSymbol();
88 final IRKeyword keyword;
89 switch (firstChild.getChildCount()) {
91 keyword = uqualKeywords.computeIfAbsent(strOf(keywordToken), Unqualified::new);
94 keyword = qualKeywords.computeIfAbsent(Map.entry(strOf(keywordToken), strOf(firstChild.getChild(2))),
95 entry -> new Qualified(entry.getKey(), entry.getValue()));
98 throw new VerifyException("Unexpected keyword " + firstChild);
101 final IRArgument argument = createArgument(stmt);
102 final ImmutableList<IRStatement> statements = createStatements(stmt);
103 final int line = keywordToken.getLine();
104 final int column = keywordToken.getCharPositionInLine();
106 switch (statements.size()) {
108 return statementOf(keyword, argument, line, column);
110 return new IRStatement144(keyword, argument, statements.get(0), line, column);
112 return new IRStatementL44(keyword, argument, statements, line, column);
116 private static @NonNull IRStatement statementOf(final IRKeyword keyword, final IRArgument argument,
117 final int line, final int column) {
118 if (line >= 0 && column >= 0) {
119 if (line <= 65535 && column <= 65535) {
120 return new IRStatement022(keyword, argument, line, column);
122 if (line <= 16777215 && column <= 255) {
123 return new IRStatement031(keyword, argument, line, column);
126 return new IRStatement044(keyword, argument, line, column);
129 private IRArgument createArgument(final StatementContext stmt) {
130 final ArgumentContext argument = stmt.argument();
131 if (argument == null) {
134 switch (argument.getChildCount()) {
136 throw new VerifyException("Unexpected shape of " + argument);
138 return createSimple(argument);
140 return createQuoted(argument);
142 return createConcatenation(argument);
146 private IRArgument createConcatenation(final ArgumentContext argument) {
147 final List<Single> parts = new ArrayList<>();
149 for (ParseTree child : argument.children) {
150 verify(child instanceof TerminalNode, "Unexpected argument component %s", child);
151 final Token token = ((TerminalNode) child).getSymbol();
152 switch (token.getType()) {
153 case YangStatementParser.SEP:
154 // Separator, just skip it over
155 case YangStatementParser.PLUS:
156 // Operator, which we are handling by concat, skip it over
157 case YangStatementParser.DQUOT_END:
158 case YangStatementParser.SQUOT_END:
159 // Quote stops, skip them over because we either already added the content, or would be appending
162 case YangStatementParser.SQUOT_STRING:
163 parts.add(createSingleQuoted(token));
165 case YangStatementParser.DQUOT_STRING:
166 parts.add(createDoubleQuoted(token));
169 throw new VerifyException("Unexpected token " + token);
173 switch (parts.size()) {
175 // A concatenation of empty strings, fall back to a single unquoted string
176 return SingleQuoted.EMPTY;
178 // A single string concatenated with empty string(s), use just the significant portion
181 // TODO: perform concatenation of single-quoted strings. For double-quoted strings this may not be as
182 // nice, but for single-quoted strings we do not need further validation in in the reactor and can
183 // use them as raw literals. This saves some indirection overhead (on memory side) and can
184 // slightly improve execution speed when we process the same IR multiple times.
185 return new Concatenation(parts);
189 private Single createQuoted(final ArgumentContext argument) {
190 final ParseTree child = argument.getChild(0);
191 verify(child instanceof TerminalNode, "Unexpected literal %s", child);
192 final Token token = ((TerminalNode) child).getSymbol();
193 switch (token.getType()) {
194 case YangStatementParser.DQUOT_STRING:
195 return createDoubleQuoted(token);
196 case YangStatementParser.SQUOT_STRING:
197 return createSingleQuoted(token);
199 throw new VerifyException("Unexpected token " + token);
203 private DoubleQuoted createDoubleQuoted(final Token token) {
204 // Whitespace normalization happens irrespective of further handling and has no effect on the result
205 final String str = intern(trimWhitespace(token.getText(), token.getCharPositionInLine() - 1));
207 // TODO: turn this into a single-quoted literal if a backslash is not present. Doing so allows the
208 // argument to be treated as a literal. See IRArgument.Single#needUnescape() for more context.
209 // This may look unimportant, but there are scenarios where we process the same AST multiple times
210 // and remembering this detail saves a string scan.
212 return dquotArguments.computeIfAbsent(str, DoubleQuoted::new);
215 private IRArgument createSimple(final ArgumentContext argument) {
216 final ParseTree child = argument.getChild(0);
217 if (child instanceof TerminalNode) {
218 final Token token = ((TerminalNode) child).getSymbol();
219 switch (token.getType()) {
220 case YangStatementParser.IDENTIFIER:
221 // This is as simple as it gets: we are dealing with an identifier here.
222 return idenArguments.computeIfAbsent(strOf(token), Identifier::new);
223 case YangStatementParser.DQUOT_END:
224 case YangStatementParser.SQUOT_END:
225 // This is an empty string, the difference between double and single quotes does not exist. Single
226 // quotes have more stringent semantics, hence use those.
227 return SingleQuoted.EMPTY;
229 throw new VerifyException("Unexpected token " + token);
233 verify(child instanceof UnquotedStringContext, "Unexpected shape of %s", argument);
234 // TODO: check non-presence of quotes and create a different subclass, so that ends up treated as if it
235 // was single-quoted, i.e. bypass the check implied by IRArgument.Single#needQuoteCheck().
236 return uquotArguments.computeIfAbsent(strOf(child), Unquoted::new);
239 private SingleQuoted createSingleQuoted(final Token token) {
240 return squotArguments.computeIfAbsent(strOf(token), SingleQuoted::new);
243 private ImmutableList<IRStatement> createStatements(final StatementContext stmt) {
244 final List<StatementContext> statements = stmt.statement();
245 return statements.isEmpty() ? ImmutableList.of()
246 : statements.stream().map(this::statementOf).collect(ImmutableList.toImmutableList());
249 private String strOf(final ParseTree tree) {
250 return intern(tree.getText());
253 private String strOf(final Token token) {
254 return intern(token.getText());
257 private String intern(final String str) {
258 return strings.computeIfAbsent(str, Function.identity());
262 static String trimWhitespace(final String str, final int dquot) {
263 final int firstBrk = str.indexOf('\n');
264 if (firstBrk == -1) {
268 // Okay, we may need to do some trimming, set up a builder and append the first segment
269 final int length = str.length();
270 final StringBuilder sb = new StringBuilder(length);
272 // Append first segment, which needs only tail-trimming
273 sb.append(str, 0, trimTrailing(str, 0, firstBrk)).append('\n');
275 // With that out of the way, setup our iteration state. The string segment we are looking at is
276 // str.substring(start, end), which is guaranteed not to include any line breaks, i.e. end <= brk unless we are
277 // at the last segment.
278 int start = firstBrk + 1;
279 int brk = str.indexOf('\n', start);
281 // Loop over inner strings
283 trimLeadingAndAppend(sb, dquot, str, start, trimTrailing(str, start, brk)).append('\n');
285 brk = str.indexOf('\n', start);
288 return trimLeadingAndAppend(sb, dquot, str, start, length).toString();
291 private static StringBuilder trimLeadingAndAppend(final StringBuilder sb, final int dquot, final String str,
292 final int start, final int end) {
296 while (pos <= dquot) {
298 // We ran out of data, nothing to append
302 final char ch = str.charAt(offset);
304 // tabs are to be treated as 8 spaces
306 } else if (WHITESPACE_MATCHER.matches(ch)) {
315 // We have expanded beyond double quotes, push equivalent spaces
316 while (pos - 1 > dquot) {
321 return sb.append(str, offset, end);
324 private static int trimTrailing(final String str, final int start, final int end) {
326 while (ret > start) {
327 final int prev = ret - 1;
328 if (!WHITESPACE_MATCHER.matches(str.charAt(prev))) {