From 04ed60d275521fd6e6ee57d5b1ce4e6a519d2e35 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Fri, 17 Apr 2020 20:01:36 +0200 Subject: [PATCH] Update YANG lexer/parser to accept free-standing '+' The lexer is not quite accurate, as it does not allow for strings starting with a '+' when not quoted. Fix this up, relaxing the rules. JIRA: YANGTOOLS-1089 Change-Id: Ifaf9c65966b43f26f37fc88a11d77fc9f43d2c8c Signed-off-by: Robert Varga --- .../yang/parser/antlr/YangStatementLexer.g4 | 2 +- .../yang/parser/antlr/YangStatementParser.g4 | 2 +- .../rfc7950/repo/ArgumentContextUtils.java | 2 + .../yangtools/yang/stmt/YT1089Test.java | 80 +++++++++++++++++++ .../src/test/resources/bugs/YT1089/foo.yang | 22 +++++ 5 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java create mode 100644 yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang diff --git a/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementLexer.g4 b/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementLexer.g4 index 8185545f38..2872acb161 100644 --- a/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementLexer.g4 +++ b/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementLexer.g4 @@ -35,7 +35,7 @@ fragment ESC : '\\' (["\\/bfnrt] | UNICODE); fragment UNICODE : 'u' HEX HEX HEX HEX; fragment HEX : [0-9a-fA-F] ; -STRING: ((~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '"' | '\'' | '}' | '/' | '+')~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '}' )* ) | SUB_STRING ); +STRING: ((~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '"' | '\'' | '}' | '/')~( '\r' | '\n' | '\t' | ' ' | ';' | '{' | '}' )* ) | SUB_STRING ); mode BLOCK_COMMENT_MODE; END_BLOCK_COMMENT : '*/' -> popMode, skip; diff --git a/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4 b/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4 index 9f836e2e32..2408d93a76 100644 --- a/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4 +++ b/yang/yang-parser-antlr/src/main/antlr4/org/opendaylight/yangtools/yang/parser/antlr/YangStatementParser.g4 @@ -14,4 +14,4 @@ options { statement : SEP* keyword SEP* (argument)? SEP* (SEMICOLON | LEFT_BRACE SEP* (statement)* SEP* RIGHT_BRACE SEP*) SEP*; keyword : (IDENTIFIER COLON)? IDENTIFIER; -argument : STRING (SEP* PLUS SEP* STRING)* | IDENTIFIER; +argument : STRING (SEP* PLUS SEP* STRING)* | PLUS | IDENTIFIER; diff --git a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java index ea51ac5149..2aefece439 100644 --- a/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java +++ b/yang/yang-parser-rfc7950/src/main/java/org/opendaylight/yangtools/yang/parser/rfc7950/repo/ArgumentContextUtils.java @@ -118,6 +118,8 @@ abstract class ArgumentContextUtils { case YangStatementParser.IDENTIFIER: // Simple case, there is a simple string, which cannot contain anything that we would need to process. return firstNode.getText(); + case YangStatementParser.PLUS: + return "+"; case YangStatementParser.STRING: // Complex case, defer to a separate method return concatStrings(context, ref); diff --git a/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java b/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java new file mode 100644 index 0000000000..cba0465bb6 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/test/java/org/opendaylight/yangtools/yang/stmt/YT1089Test.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2020 PANTHEON.tech, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.stmt; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import com.google.common.collect.Iterables; +import java.util.Iterator; +import org.junit.Test; +import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext; +import org.opendaylight.yangtools.yang.model.api.meta.EffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.ContactEffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.DescriptionEffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.LeafEffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.NamespaceEffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.OrganizationEffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.PrefixEffectiveStatement; +import org.opendaylight.yangtools.yang.model.api.stmt.TypeEffectiveStatement; + +public class YT1089Test { + @Test + public void testPlusLexing() throws Exception { + final EffectiveModelContext ctx = StmtTestUtils.parseYangSource("/bugs/YT1089/foo.yang"); + assertEquals(1, ctx.getModuleStatements().size()); + + final Iterator> it = + Iterables.getOnlyElement(ctx.getModuleStatements().values()).effectiveSubstatements().iterator(); + + assertThat(it.next(), instanceOf(NamespaceEffectiveStatement.class)); + assertThat(it.next(), instanceOf(PrefixEffectiveStatement.class)); + + EffectiveStatement stmt = it.next(); + assertThat(stmt, instanceOf(DescriptionEffectiveStatement.class)); + assertEquals("+something", stmt.argument()); + + stmt = it.next(); + assertThat(stmt, instanceOf(ContactEffectiveStatement.class)); + assertEquals("contact++", stmt.argument()); + + stmt = it.next(); + assertThat(stmt, instanceOf(OrganizationEffectiveStatement.class)); + assertEquals("organiza++tion", stmt.argument()); + + assertFoo(it.next()); + assertBar(it.next()); + assertFalse(it.hasNext()); + } + + private static void assertFoo(final EffectiveStatement stmt) { + assertThat(stmt, instanceOf(LeafEffectiveStatement.class)); + assertEquals(QName.create("urn:foo", "foo"), stmt.argument()); + + final Iterator> it = stmt.effectiveSubstatements().iterator(); + assertThat(it.next(), instanceOf(TypeEffectiveStatement.class)); + assertEquals("+", it.next().argument()); + assertEquals("squotdquot", it.next().argument()); + assertEquals("++", it.next().argument()); + assertFalse(it.hasNext()); + } + + private static void assertBar(final EffectiveStatement stmt) { + assertThat(stmt, instanceOf(LeafEffectiveStatement.class)); + assertEquals(QName.create("urn:foo", "bar"), stmt.argument()); + final Iterator> it = stmt.effectiveSubstatements().iterator(); + assertThat(it.next(), instanceOf(TypeEffectiveStatement.class)); + assertEquals("++", it.next().argument()); + assertEquals("+ + ++", it.next().argument()); + assertEquals("++ + +", it.next().argument()); + assertFalse(it.hasNext()); + } +} diff --git a/yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang b/yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang new file mode 100644 index 0000000000..982dca3269 --- /dev/null +++ b/yang/yang-parser-rfc7950/src/test/resources/bugs/YT1089/foo.yang @@ -0,0 +1,22 @@ +module foo { + namespace urn:foo; + prefix foo; + + description +something; + contact contact++; + organization organiza++tion; + + leaf foo { + type string; + default +; + description 'squot' + "dquot"; + reference "+" + '+'; + } + + leaf bar { + type string; + default ++; + description "+ + ++"; + reference "++ + +"; + } +} -- 2.36.6