import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.CharMatcher;
import com.google.common.base.VerifyException;
-import java.util.regex.Pattern;
+import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.eclipse.jdt.annotation.NonNull;
import org.opendaylight.yangtools.yang.common.YangVersion;
import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser;
import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.ArgumentContext;
+import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.QuotedStringContext;
+import org.opendaylight.yangtools.yang.parser.antlr.YangStatementParser.UnquotedStringContext;
import org.opendaylight.yangtools.yang.parser.spi.source.SourceException;
import org.opendaylight.yangtools.yang.parser.spi.source.StatementSourceReference;
private static final @NonNull RFC6020 INSTANCE = new RFC6020();
@Override
- void checkDoubleQuoted(final String str, final StatementSourceReference ref) {
+ void checkDoubleQuoted(final String str, final StatementSourceReference ref, final int backslash) {
// No-op
}
private static final @NonNull RFC7950 INSTANCE = new RFC7950();
@Override
- void checkDoubleQuoted(final String str, final StatementSourceReference ref) {
- // FIXME: YANGTOOLS-1079: we should forward backslash to this method, so that it does not start from the
- // start from the start of the string. Furthermore this logic should operate on spans
- // of characters -- i.e. the check for backslash should be a search instead -- as
- // String knows how to do that and can do it more efficiently than this loop.
- for (int i = 0; i < str.length() - 1; i++) {
- if (str.charAt(i) == '\\') {
- switch (str.charAt(i + 1)) {
+ void checkDoubleQuoted(final String str, final StatementSourceReference ref, final int backslash) {
+ if (backslash < str.length() - 1) {
+ int index = backslash;
+ while (index != -1) {
+ switch (str.charAt(index + 1)) {
case 'n':
case 't':
case '\\':
case '\"':
- i++;
+ index = str.indexOf('\\', index + 2);
break;
default:
throw new SourceException(ref, "YANG 1.1: illegal double quoted string (%s). In double "
- + "quoted string the backslash must be followed by one of the following character "
- + "[n,t,\",\\], but was '%s'.", str, str.charAt(i + 1));
+ + "quoted string the backslash must be followed by one of the following character "
+ + "[n,t,\",\\], but was '%s'.", str, str.charAt(index + 1));
}
}
}
}
private static final CharMatcher WHITESPACE_MATCHER = CharMatcher.whitespace();
- private static final Pattern ESCAPED_DQUOT = Pattern.compile("\\\"", Pattern.LITERAL);
- private static final Pattern ESCAPED_BACKSLASH = Pattern.compile("\\\\", Pattern.LITERAL);
- private static final Pattern ESCAPED_LF = Pattern.compile("\\n", Pattern.LITERAL);
- private static final Pattern ESCAPED_TAB = Pattern.compile("\\t", Pattern.LITERAL);
private ArgumentContextUtils() {
// Hidden on purpose
final @NonNull String stringFromStringContext(final ArgumentContext context, final StatementSourceReference ref) {
// Get first child, which we fully expect to exist and be a lexer token
final ParseTree firstChild = context.getChild(0);
- verify(firstChild instanceof TerminalNode, "Unexpected shape of %s", context);
- final TerminalNode firstNode = (TerminalNode) firstChild;
- final int firstType = firstNode.getSymbol().getType();
- switch (firstType) {
- case YangStatementParser.IDENTIFIER:
- // Simple case, there is a simple string, which cannot contain anything that we would need to process.
- return firstNode.getText();
- case YangStatementParser.STRING:
- // Complex case, defer to a separate method
- return concatStrings(context, ref);
+ if (firstChild instanceof UnquotedStringContext) {
+ // Simple case, just grab the text, as ANTLR has done all the heavy lifting
+ final String str = firstChild.getText();
+ checkUnquoted(str, ref);
+ return str;
+ }
+
+ verify(firstChild instanceof QuotedStringContext, "Unexpected shape of %s", context);
+ if (context.getChildCount() == 1) {
+ // No concatenation needed, special-case
+ return unquoteString((QuotedStringContext) firstChild, ref);
+ }
+
+ // Potentially-complex case of string quoting, escaping and concatenation.
+ return concatStrings(context, ref);
+ }
+
+ private String unquoteString(final QuotedStringContext context, final StatementSourceReference ref) {
+ final ParseTree secondChild = context.getChild(1);
+ verify(secondChild instanceof TerminalNode, "Unexpected shape of %s", context);
+ final Token secondToken = ((TerminalNode) secondChild).getSymbol();
+ final int type = secondToken.getType();
+ switch (type) {
+ case YangStatementParser.DQUOT_END:
+ case YangStatementParser.SQUOT_END:
+ // We are missing actual body, hence this is an empty string
+ return "";
+ case YangStatementParser.SQUOT_STRING:
+ return secondChild.getText();
+ case YangStatementParser.DQUOT_STRING:
+ // We should be looking at the first token, which is DQUOT_START, but since it is a single-character
+ // token, let's not bother.
+ return normalizeDoubleQuoted(secondChild.getText(), secondToken.getCharPositionInLine() - 1, ref);
default:
- throw new VerifyException("Unexpected first symbol in " + context);
+ throw new VerifyException("Unhandled token type " + type);
}
}
/*
* We have multiple fragments. Just search the tree. This code is equivalent to
*
- * context.STRING().forEach(stringNode -> appendString(sb, stringNode, ref))
+ * context.quotedString().forEach(stringNode -> sb.append(unquoteString(stringNode, ref))
*
* except we minimize allocations which that would do.
*/
final StringBuilder sb = new StringBuilder();
for (ParseTree child : context.children) {
- verify(child instanceof TerminalNode, "Unexpected fragment component %s", child);
- final TerminalNode childNode = (TerminalNode) child;
- switch (childNode.getSymbol().getType()) {
- case YangStatementParser.SEP:
- // Ignore whitespace
- break;
- case YangStatementParser.PLUS:
- // Operator, which we are handling by concat
- break;
- case YangStatementParser.STRING:
- // a lexer string, could be pretty much anything
- // FIXME: YANGTOOLS-1079: appendString() is a dispatch based on quotes, which we should be able to
- // defer to lexer for a dedicated type. That would expand the switch table
- // here, but since we have it anyway, it would be nice to have the quoting
- // distinction already taken care of. The performance difference will need to
- // be benchmarked, though.
- appendString(sb, childNode, ref);
- break;
- default:
- throw new VerifyException("Unexpected symbol in " + childNode);
+ if (child instanceof TerminalNode) {
+ final TerminalNode childNode = (TerminalNode) child;
+ switch (childNode.getSymbol().getType()) {
+ case YangStatementParser.SEP:
+ case YangStatementParser.PLUS:
+ // Operator, which we are handling by concat
+ break;
+ default:
+ throw new VerifyException("Unexpected symbol in " + childNode);
+ }
+ } else {
+ verify(child instanceof QuotedStringContext, "Unexpected fragment component %s", child);
+ sb.append(unquoteString((QuotedStringContext) child, ref));
+ continue;
}
}
return sb.toString();
}
- private void appendString(final StringBuilder sb, final TerminalNode stringNode,
- final StatementSourceReference ref) {
- final String str = stringNode.getText();
- final char firstChar = str.charAt(0);
- final char lastChar = str.charAt(str.length() - 1);
- if (firstChar == '"' && lastChar == '"') {
- sb.append(normalizeDoubleQuoted(str.substring(1, str.length() - 1),
- stringNode.getSymbol().getCharPositionInLine(), ref));
- } else if (firstChar == '\'' && lastChar == '\'') {
- /*
- * According to RFC6020 a single quote character cannot occur in a single-quoted string, even when preceded
- * by a backslash.
- */
- sb.append(str, 1, str.length() - 1);
- } else {
- checkUnquoted(str, ref);
- sb.append(str);
- }
- }
-
private String normalizeDoubleQuoted(final String str, final int dquot, final StatementSourceReference ref) {
// Whitespace normalization happens irrespective of further handling and has no effect on the result
final String stripped = trimWhitespace(str, dquot);
// Now we need to perform some amount of unescaping. This serves as a pre-check before we dispatch
// validation and processing (which will reuse the work we have done)
final int backslash = stripped.indexOf('\\');
- return backslash == -1 ? stripped : unescape(stripped, backslash, ref);
+ return backslash == -1 ? stripped : unescape(ref, stripped, backslash);
}
/*
* account the for it with lexer modes. We do not want to force a re-lexing phase in the parser just because
* we decided to let ANTLR do the work.
*/
- // FIXME: YANGTOOLS-1079: Re-evaluate above comment once our integration surface with lexer has been decided
- abstract void checkDoubleQuoted(String str, StatementSourceReference ref);
+ abstract void checkDoubleQuoted(String str, StatementSourceReference ref, int backslash);
abstract void checkUnquoted(String str, StatementSourceReference ref);
/*
* Unescape escaped double quotes, tabs, new line and backslash in the inner string and trim the result.
*/
- private String unescape(final String str, final int backslash, final StatementSourceReference ref) {
- checkDoubleQuoted(str, ref);
-
- // FIXME: YANGTOOLS-1079: given we the leading backslash, it would be more efficient to walk the string and
- // unescape in one go
- return ESCAPED_TAB.matcher(
- ESCAPED_LF.matcher(
- ESCAPED_BACKSLASH.matcher(
- ESCAPED_DQUOT.matcher(str).replaceAll("\\\""))
- .replaceAll("\\\\"))
- .replaceAll("\\\n"))
- .replaceAll("\\\t");
+ private String unescape(final StatementSourceReference ref, final String str, final int backslash) {
+ checkDoubleQuoted(str, ref, backslash);
+ StringBuilder sb = new StringBuilder(str.length());
+ unescapeBackslash(sb, str, backslash);
+ return sb.toString();
+ }
+
+ @VisibleForTesting
+ static void unescapeBackslash(final StringBuilder sb, final String str, final int backslash) {
+ String substring = str;
+ int backslashIndex = backslash;
+ while (true) {
+ int nextIndex = backslashIndex + 1;
+ if (backslashIndex != -1 && nextIndex < substring.length()) {
+ replaceBackslash(sb, substring, nextIndex);
+ substring = substring.substring(nextIndex + 1);
+ if (substring.length() > 0) {
+ backslashIndex = substring.indexOf('\\');
+ } else {
+ break;
+ }
+ } else {
+ sb.append(substring);
+ break;
+ }
+ }
+ }
+
+ private static void replaceBackslash(final StringBuilder sb, final String str, final int nextAfterBackslash) {
+ int backslash = nextAfterBackslash - 1;
+ sb.append(str, 0, backslash);
+ final char c = str.charAt(nextAfterBackslash);
+ switch (c) {
+ case '\\':
+ case '"':
+ sb.append(c);
+ break;
+ case 't':
+ sb.append('\t');
+ break;
+ case 'n':
+ sb.append('\n');
+ break;
+ default:
+ sb.append(str, backslash, nextAfterBackslash + 1);
+ }
}
@VisibleForTesting
Code Review / yangtools.git / blobdiff