Rework YANG lexer/parser

[yangtools.git] / yang / yang-parser-antlr / src / main / antlr4 / org / opendaylight / yangtools / yang / parser / antlr / YangStatementParser.g4

diff --git a/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4 b/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4
index 9f836e2e3210053ccbc39cda5eaccdcfcde9111d..cbfeee38c7b5211477ff370d074508732fb5f841 100644 (file)
--- a/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4
+++ b/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4
@@ -11,7 +11,34 @@ options {
     tokenVocab = YangStatementLexer;
 }
 
-statement : SEP* keyword SEP* (argument)? SEP* (SEMICOLON | LEFT_BRACE SEP* (statement)* SEP* RIGHT_BRACE SEP*) SEP*;
-keyword : (IDENTIFIER COLON)? IDENTIFIER;
+// NOTE: we need to use SEP*/SEP+ because comments end up breaking whitespace
+//       sequences into two.
+file : SEP* statement SEP* EOF;
+statement : keyword (SEP+ argument)? SEP* (SEMICOLON | LEFT_BRACE SEP* (statement SEP*)* RIGHT_BRACE);
+keyword : IDENTIFIER (COLON IDENTIFIER)?;
+
+// Alright, so what constitutes a string is rather funky. We need to deal with
+// the flaky definitions of RFC6020, which allow for insane quoting as well as
+// exclusion of comments. We also need to allow for stitching back tokens like
+// PLUS/COLON, which may end up being valid identifiers. Finally we need to allow
+// IDENTIFIER to be concatenated back to a string
+argument : unquotedString | quotedString (SEP* PLUS SEP* quotedString)*;
+
+quotedString :
+    DQUOT_START DQUOT_STRING? DQUOT_END
+    |
+    SQUOT_START SQUOT_STRING? SQUOT_END
+    ;
+
+unquotedString : SLASH | STAR+ | (SLASH? | STAR*) stringPart+ (SLASH? | STAR*);
+
+// A string which is guaranteed to not have slash/star in either start or end
+// and can thus be concatenated without allowing '/*', '//' and '*/' to appear.
+stringPart:
+    (IDENTIFIER | COLON | PLUS | UQUOT_STRING)+
+    |
+    stringPart SLASH stringPart
+    |
+    stringPart STAR+ stringPart
+    ;
 
-argument : STRING (SEP* PLUS SEP* STRING)* | IDENTIFIER;