Strings Ending in Backslash are Improperly Lexed

Using greedy match for SUB_STRING fragment's forward lookahead
is troublesome because it will consume all non-double-quotes
until the final one -- including any escapes present.

This patch fixes the lexer to use non-greedy subrule, so that
escapes are properly accounted for.

JIRA: YANGTOOLS-971
Change-Id: Ic93c5c2bfa807001735125bd0367c29a1a25ea99
Signed-off-by: Danielle Mustillo <danielle.mustillo@ericsson.com>
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>
(cherry picked from commit e307e3f6893f94bc0a6b3fac094270f3e206eef2)

diff --git a/yang/yang-parser-rfc7950/src/main/antlr/YangStatementLexer.g4 b/yang/yang-parser-rfc7950/src/main/antlr/YangStatementLexer.g4
index b17c6674f221998d793015366841f86747719a75..3ddb6521ecb0552e675685f637b63971b45b72b4 100644 (file)
--- a/yang/yang-parser-rfc7950/src/main/antlr/YangStatementLexer.g4
+++ b/yang/yang-parser-rfc7950/src/main/antlr/YangStatementLexer.g4
@@ -34,7 +34,7 @@ START_BLOCK_COMMENT : '/*' ->pushMode(BLOCK_COMMENT_MODE), skip;
 SEP: [ \n\r\t]+ -> type(SEP);
 IDENTIFIER : [a-zA-Z_/][a-zA-Z0-9_\-.:/]* -> type(IDENTIFIER);
 
-fragment SUB_STRING : ('"' (ESC | ~["])*'"') | ('\'' (ESC | ~['])*'\'');
+fragment SUB_STRING : ('"' (ESC | ~["])*? '"') | ('\'' (ESC | ~['])* '\'');
 fragment ESC : '\\' (["\\/bfnrt] | UNICODE);
 fragment UNICODE : 'u' HEX HEX HEX HEX;
 fragment HEX : [0-9a-fA-F] ;
@@ -43,4 +43,4 @@ STRING: ((~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '"' | '\'' | '}' | '/' | '+'
 
 mode BLOCK_COMMENT_MODE;
 END_BLOCK_COMMENT : '*/' -> popMode, skip;
-BLOCK_COMMENT :  . -> skip;
\ No newline at end of file
+BLOCK_COMMENT :  . -> skip;

diff --git a/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT971Test.java b/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT971Test.java
new file mode 100644 (file)
index 0000000..ae5fedc
--- /dev/null
+++ b/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT971Test.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 Ericsson AB. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang.stmt;
+
+import static org.hamcrest.core.IsInstanceOf.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Optional;
+import org.junit.Test;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.common.QNameModule;
+import org.opendaylight.yangtools.yang.common.Revision;
+import org.opendaylight.yangtools.yang.model.api.ContainerSchemaNode;
+import org.opendaylight.yangtools.yang.model.api.DataSchemaNode;
+import org.opendaylight.yangtools.yang.model.api.LeafSchemaNode;
+import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+import org.opendaylight.yangtools.yang.model.api.type.Int16TypeDefinition;
+import org.opendaylight.yangtools.yang.model.api.type.Int32TypeDefinition;
+import org.opendaylight.yangtools.yang.model.parser.api.YangSyntaxErrorException;
+import org.opendaylight.yangtools.yang.parser.spi.meta.ReactorException;
+
+public class YT971Test {
+    private static final QNameModule NAMESPACE = QNameModule.create(URI.create("test"), Revision.of("2019-03-25"));
+
+    @Test
+    public void testEscapeLexer() throws URISyntaxException, IOException, YangSyntaxErrorException, ReactorException {
+        final SchemaContext schemaContext = StmtTestUtils.parseYangSource("/bugs/YT971/test.yang");
+        assertNotNull(schemaContext);
+
+        final DataSchemaNode someContainer = schemaContext.findDataChildByName(
+            QName.create(NAMESPACE, "some-container")).get();
+        assertThat(someContainer, instanceOf(ContainerSchemaNode.class));
+        final ContainerSchemaNode containerSchemaNode = (ContainerSchemaNode) someContainer;
+
+        final DataSchemaNode someLeaf = containerSchemaNode.findDataChildByName(QName.create(NAMESPACE, "some-leaf"))
+                .get();
+        assertThat(someLeaf, instanceOf(LeafSchemaNode.class));
+        final LeafSchemaNode leafSchemaNode = (LeafSchemaNode) someLeaf;
+        assertEquals(Optional.of("Some string that ends with a backslash (with escape backslash too) \\"),
+                     leafSchemaNode.getDescription());
+        assertThat(leafSchemaNode.getType(), instanceOf(Int16TypeDefinition.class));
+
+        final DataSchemaNode someOtherLeaf = containerSchemaNode.findDataChildByName(
+                QName.create(NAMESPACE, "some-other-leaf")).get();
+        assertThat(someOtherLeaf, instanceOf(LeafSchemaNode.class));
+
+        final LeafSchemaNode otherLeafSchemaNode = (LeafSchemaNode) someOtherLeaf;
+        assertEquals(Optional.of("Some string after the double backslash"), otherLeafSchemaNode.getDescription());
+        assertThat(otherLeafSchemaNode.getType(), instanceOf(Int32TypeDefinition.class));
+    }
+}

diff --git a/yang/yang-parser-rfc7950/src/test/resources/bugs/YT971/test.yang b/yang/yang-parser-rfc7950/src/test/resources/bugs/YT971/test.yang
new file mode 100644 (file)
index 0000000..c105d8a
--- /dev/null
+++ b/yang/yang-parser-rfc7950/src/test/resources/bugs/YT971/test.yang
@@ -0,0 +1,18 @@
+module test {
+    namespace test;
+    prefix test;
+    yang-version 1.1;
+
+    revision 2019-03-25;
+
+    container some-container {
+        leaf some-leaf {
+            description "Some string that ends with a backslash (with escape backslash too) \\";
+            type int16;
+        }
+        leaf some-other-leaf {
+            description "Some string after the double backslash";
+            type int32;
+        }
+    }
+}