Update YANG lexer/parser to accept free-standing '+'

The lexer is not quite accurate, as it does not allow for strings
starting with a '+' when not quoted. Fix this up, relaxing the
rules.

JIRA: YANGTOOLS-1089
Change-Id: Ifaf9c65966b43f26f37fc88a11d77fc9f43d2c8c
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>
(cherry picked from commit 04ed60d275521fd6e6ee57d5b1ce4e6a519d2e35)

diff --git a/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementLexer.g4 b/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementLexer.g4
index 8185545f38b047a9d728420be7ac2c6e0fcf978f..2872acb1616fc5d9279481439870a73a026695fc 100644 (file)
--- a/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementLexer.g4
+++ b/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementLexer.g4
@@ -35,7 +35,7 @@ fragment ESC : '\\' (["\\/bfnrt] | UNICODE);
 fragment UNICODE : 'u' HEX HEX HEX HEX;
 fragment HEX : [0-9a-fA-F] ;
 
-STRING: ((~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '"' | '\'' | '}' | '/' | '+')~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '}' )* ) | SUB_STRING );
+STRING: ((~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '"' | '\'' | '}' | '/')~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '}' )* ) | SUB_STRING );
 
 mode BLOCK_COMMENT_MODE;
 END_BLOCK_COMMENT : '*/' -> popMode, skip;

diff --git a/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementParser.g4 b/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementParser.g4
index 9f836e2e3210053ccbc39cda5eaccdcfcde9111d..2408d93a765a09dd54d297335d69ccffae476335 100644 (file)
--- a/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementParser.g4
+++ b/yang/yang-parser-rfc7950/src/main/antlr/org/opendaylight/yangtools/antlrv4/code/gen/YangStatementParser.g4
@@ -14,4 +14,4 @@ options {
 statement : SEP* keyword SEP* (argument)? SEP* (SEMICOLON | LEFT_BRACE SEP* (statement)* SEP* RIGHT_BRACE SEP*) SEP*;
 keyword : (IDENTIFIER COLON)? IDENTIFIER;
 
-argument : STRING (SEP* PLUS SEP* STRING)* | IDENTIFIER;
+argument : STRING (SEP* PLUS SEP* STRING)* | PLUS | IDENTIFIER;

diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java
index 530875cbcbbd0d2e08fbc2e64374744e9ca794f6..5cf955a930de6377eddbf5150aefdb5a85b0da05 100644 (file)
--- a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java
+++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java
@@ -118,6 +118,8 @@ abstract class ArgumentContextUtils {
             case YangStatementParser.IDENTIFIER:
                 // Simple case, there is a simple string, which cannot contain anything that we would need to process.
                 return firstNode.getText();
+            case YangStatementParser.PLUS:
+                return "+";
             case YangStatementParser.STRING:
                 // Complex case, defer to a separate method
                 return concatStrings(context, ref);

diff --git a/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java b/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java
new file mode 100644 (file)
index 0000000..690e53d
--- /dev/null
+++ b/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.stmt;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import com.google.common.collect.Iterables;
+import java.util.Iterator;
+import org.junit.Test;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+import org.opendaylight.yangtools.yang.model.api.meta.EffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.ContactEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.DescriptionEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.LeafEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.ModuleEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.NamespaceEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.OrganizationEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.PrefixEffectiveStatement;
+import org.opendaylight.yangtools.yang.model.api.stmt.TypeEffectiveStatement;
+
+public class YT1089Test {
+    @Test
+    public void testPlusLexing() throws Exception {
+        final SchemaContext ctx = StmtTestUtils.parseYangSource("/bugs/YT1089/foo.yang");
+        assertEquals(1, ctx.getModules().size());
+
+        final Iterator<? extends EffectiveStatement<?, ?>> it = ((ModuleEffectiveStatement)
+                Iterables.getOnlyElement(ctx.getModules())).effectiveSubstatements().iterator();
+
+        assertThat(it.next(), instanceOf(NamespaceEffectiveStatement.class));
+        assertThat(it.next(), instanceOf(PrefixEffectiveStatement.class));
+
+        EffectiveStatement<?, ?> stmt = it.next();
+        assertThat(stmt, instanceOf(DescriptionEffectiveStatement.class));
+        assertEquals("+something", stmt.argument());
+
+        stmt = it.next();
+        assertThat(stmt, instanceOf(ContactEffectiveStatement.class));
+        assertEquals("contact++", stmt.argument());
+
+        stmt = it.next();
+        assertThat(stmt, instanceOf(OrganizationEffectiveStatement.class));
+        assertEquals("organiza++tion", stmt.argument());
+
+        assertFoo(it.next());
+        assertBar(it.next());
+        assertFalse(it.hasNext());
+    }
+
+    private static void assertFoo(final EffectiveStatement<?, ?> stmt) {
+        assertThat(stmt, instanceOf(LeafEffectiveStatement.class));
+        assertEquals(QName.create("urn:foo", "foo"), stmt.argument());
+
+        final Iterator<? extends EffectiveStatement<?, ?>> it = stmt.effectiveSubstatements().iterator();
+        assertThat(it.next(), instanceOf(TypeEffectiveStatement.class));
+        assertEquals("+", it.next().argument());
+        assertEquals("squotdquot", it.next().argument());
+        assertEquals("++", it.next().argument());
+        assertFalse(it.hasNext());
+    }
+
+    private static void assertBar(final EffectiveStatement<?, ?> stmt) {
+        assertThat(stmt, instanceOf(LeafEffectiveStatement.class));
+        assertEquals(QName.create("urn:foo", "bar"), stmt.argument());
+        final Iterator<? extends EffectiveStatement<?, ?>> it = stmt.effectiveSubstatements().iterator();
+        assertThat(it.next(), instanceOf(TypeEffectiveStatement.class));
+        assertEquals("++", it.next().argument());
+        assertEquals("+ + ++", it.next().argument());
+        assertEquals("++ + +", it.next().argument());
+        assertFalse(it.hasNext());
+    }
+}

diff --git a/yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang b/yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang
new file mode 100644 (file)
index 0000000..982dca3
--- /dev/null
+++ b/yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang
@@ -0,0 +1,22 @@
+module foo {
+    namespace urn:foo;
+    prefix foo;
+
+    description +something;
+    contact contact++;
+    organization organiza++tion;
+
+    leaf foo {
+      type string;
+      default +;
+      description 'squot' + "dquot";
+      reference "+" + '+';
+    }
+
+    leaf bar {
+      type string;
+      default ++;
+      description "+ + ++";
+      reference "++ + +";
+    }
+}