Maven plugin ignores exact YANG duplicates in dependencies

Maven plugin failed if same YANG file was available
in two different artifacts available via transitive dependencies
which actually prevented use-cases as non-breaking
artifact renaming.

Change-Id: I88dee0470568910c35e4f310bf850605dea1ca4c
Signed-off-by: Peter Kajsa <pkajsa@cisco.com>
Signed-off-by: Tony Tkacik <ttkacik@cisco.com>

diff --git a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/Util.java b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/Util.java
index ee9a6897423788f38eeff1bbce37a10103f44411..caac54281e98dff5390fdbdd4ed3a3ff84fbb70e 100644 (file)
--- a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/Util.java
+++ b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/Util.java
@@ -7,12 +7,14 @@
  */
 package org.opendaylight.yangtools.yang2sources.plugin;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import java.io.Closeable;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -24,7 +26,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.maven.artifact.Artifact;
 import org.apache.maven.artifact.repository.ArtifactRepository;
@@ -35,14 +36,9 @@ import org.apache.maven.model.Plugin;
 import org.apache.maven.plugin.MojoFailureException;
 import org.apache.maven.plugin.logging.Log;
 import org.apache.maven.project.MavenProject;
+import org.apache.maven.repository.RepositorySystem;
 import org.opendaylight.yangtools.yang.model.api.Module;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
-import org.opendaylight.yangtools.yang.parser.util.NamedFileInputStream;
-import org.apache.maven.repository.RepositorySystem;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
 
 final class Util {
 
@@ -256,32 +252,29 @@ final class Util {
     }
 
     static final class YangsInZipsResult implements Closeable {
-        private final List<InputStream> yangStreams;
+        private final List<YangSourceFromDependency> yangStreams;
         private final List<Closeable> zipInputStreams;
 
-        private YangsInZipsResult(List<InputStream> yangStreams, List<Closeable> zipInputStreams) {
+        private YangsInZipsResult(List<YangSourceFromDependency> yangStreams, List<Closeable> zipInputStreams) {
             this.yangStreams = yangStreams;
             this.zipInputStreams = zipInputStreams;
         }
 
         @Override
         public void close() throws IOException {
-            for (InputStream is : yangStreams) {
-                is.close();
-            }
             for (Closeable is : zipInputStreams) {
                 is.close();
             }
         }
 
-        public List<InputStream> getYangStreams() {
+        public List<YangSourceFromDependency> getYangStreams() {
             return this.yangStreams;
         }
     }
 
     static YangsInZipsResult findYangFilesInDependenciesAsStream(Log log, MavenProject project)
             throws MojoFailureException {
-        List<InputStream> yangsFromDependencies = new ArrayList<>();
+        List<YangSourceFromDependency> yangsFromDependencies = new ArrayList<>();
         List<Closeable> zips = new ArrayList<>();
         try {
             List<File> filesOnCp = Util.getClassPath(project);
@@ -301,8 +294,8 @@ final class Util {
                                 return name.endsWith(".yang") && new File(dir, name).isFile();
                             }
                         });
-                        for (File yangFile : yangFiles) {
-                            yangsFromDependencies.add(new NamedFileInputStream(yangFile, YangToSourcesProcessor.META_INF_YANG_STRING + File.separator + yangFile.getName()));
+                        for (final File yangFile : yangFiles) {
+                            yangsFromDependencies.add(new YangSourceFromFile(file));
                         }
                     }
 
@@ -318,10 +311,7 @@ final class Util {
                         if (entryName.startsWith(YangToSourcesProcessor.META_INF_YANG_STRING_JAR)
                                 && !entry.isDirectory() && entryName.endsWith(".yang")) {
                             foundFilesForReporting.add(entryName);
-                            // This will be closed after all streams are
-                            // parsed.
-                            InputStream entryStream = zip.getInputStream(entry);
-                            yangsFromDependencies.add(entryStream);
+                            yangsFromDependencies.add(new YangSourceInZipFile(zip, entry));
                         }
                     }
                 }
@@ -337,6 +327,18 @@ final class Util {
         return new YangsInZipsResult(yangsFromDependencies, zips);
     }
 
+    /**
+     * Find all dependencies which contains yang sources
+     *
+     * Returns collection of YANG files and Zip files which contains YANG files.
+     *
+     * FIXME: Rename to what class is actually doing.
+     *
+     * @param log
+     * @param project
+     * @return
+     * @throws MojoFailureException
+     */
     static Collection<File> findYangFilesInDependencies(Log log, MavenProject project) throws MojoFailureException {
         final List<File> yangsFilesFromDependencies = new ArrayList<>();
 

diff --git a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceFromDependency.java b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceFromDependency.java
new file mode 100644 (file)
index 0000000..6a53153
--- /dev/null
+++ b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceFromDependency.java
@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang2sources.plugin;
+
+import com.google.common.io.ByteSource;
+
+abstract class YangSourceFromDependency extends ByteSource {
+}

diff --git a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceFromFile.java b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceFromFile.java
new file mode 100644 (file)
index 0000000..a5d15e8
--- /dev/null
+++ b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceFromFile.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang2sources.plugin;
+
+import com.google.common.base.Preconditions;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import org.opendaylight.yangtools.yang.parser.util.NamedFileInputStream;
+
+class YangSourceFromFile extends YangSourceFromDependency {
+
+    private final File source;
+
+    public YangSourceFromFile(File source) {
+        this.source = Preconditions.checkNotNull(source);
+    }
+
+    @Override
+    public InputStream openStream() throws IOException {
+
+        return new NamedFileInputStream(source, YangToSourcesProcessor.META_INF_YANG_STRING + File.separator
+                + source.getName());
+    }
+
+    @Override
+    public long size() throws IOException {
+        return source.length();
+    }
+
+}

diff --git a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceInZipFile.java b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceInZipFile.java
new file mode 100644 (file)
index 0000000..eb876f6
--- /dev/null
+++ b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangSourceInZipFile.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang2sources.plugin;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+class YangSourceInZipFile extends YangSourceFromDependency {
+
+    private final ZipFile file;
+    private final ZipEntry entry;
+
+    YangSourceInZipFile(ZipFile file, ZipEntry entry) {
+        this.file = Preconditions.checkNotNull(file);
+        this.entry = Preconditions.checkNotNull(entry);
+    }
+
+    @Override
+    public long size() {
+        return entry.getSize();
+    }
+
+    @Override
+    public InputStream openStream() throws IOException {
+        return file.getInputStream(entry);
+    }
+}
\ No newline at end of file

diff --git a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java
index 0402e8b8a36517b6378e1bb2082a23208adbfe64..d939ccfa34875163a2f35e890bac66af1f6cc29e 100644 (file)
--- a/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java
+++ b/yang/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java
@@ -20,6 +20,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.commons.io.IOUtils;
 import org.apache.maven.model.Resource;
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugin.MojoFailureException;
@@ -28,6 +30,7 @@ import org.apache.maven.project.MavenProject;
 import org.opendaylight.yangtools.yang.model.api.Module;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 import org.opendaylight.yangtools.yang.parser.impl.YangParserImpl;
+import org.opendaylight.yangtools.yang.parser.repo.URLSchemaContextResolver;
 import org.opendaylight.yangtools.yang.parser.util.NamedFileInputStream;
 import org.opendaylight.yangtools.yang2sources.plugin.ConfigArg.CodeGeneratorArg;
 import org.opendaylight.yangtools.yang2sources.plugin.Util.ContextHolder;
@@ -52,6 +55,7 @@ class YangToSourcesProcessor {
     private final boolean inspectDependencies;
     private final BuildContext buildContext;
     private final YangProvider yangProvider;
+    private final URLSchemaContextResolver resolver;
 
     @VisibleForTesting
     YangToSourcesProcessor(Log log, File yangFilesRootDir, File[] excludedFiles, List<CodeGeneratorArg> codeGenerators,
@@ -73,6 +77,7 @@ class YangToSourcesProcessor {
         this.project = Util.checkNotNull(project, "project");
         this.inspectDependencies = inspectDependencies;
         this.yangProvider = yangProvider;
+        this.resolver = URLSchemaContextResolver.create("maven-plugin");
     }
 
     YangToSourcesProcessor(BuildContext buildContext, Log log, File yangFilesRootDir, File[] excludedFiles, List<CodeGeneratorArg> codeGenerators,
@@ -99,6 +104,8 @@ class YangToSourcesProcessor {
              * dependencies.
              */
             final Collection<File> yangFilesInProject = Util.listFiles(yangFilesRootDir, excludedFiles, log);
+
+
             final Collection<File> allFiles = new ArrayList<>(yangFilesInProject);
             if (inspectDependencies) {
                 allFiles.addAll(Util.findYangFilesInDependencies(log, project));
@@ -128,6 +135,7 @@ class YangToSourcesProcessor {
 
             final List<InputStream> yangsInProject = new ArrayList<>();
             for (final File f : yangFilesInProject) {
+                // FIXME: This is hack - normal path should be reported.
                 yangsInProject.add(new NamedFileInputStream(f, META_INF_YANG_STRING + File.separator + f.getName()));
             }
 
@@ -147,7 +155,9 @@ class YangToSourcesProcessor {
                     YangsInZipsResult dependentYangResult = Util.findYangFilesInDependenciesAsStream(log, project);
                     Closeable dependentYangResult1 = dependentYangResult;
                     closeables.add(dependentYangResult1);
-                    all.addAll(dependentYangResult.getYangStreams());
+                    List<InputStream> yangStreams = toStreamsWithoutDuplicates(dependentYangResult.getYangStreams());
+                    all.addAll(yangStreams);
+                    closeables.addAll(yangStreams);
                 }
 
                 allYangModules = parser.parseYangModelsFromStreamsMapped(all);
@@ -180,6 +190,23 @@ class YangToSourcesProcessor {
         }
     }
 
+    private List<InputStream> toStreamsWithoutDuplicates(List<YangSourceFromDependency> list) throws IOException {
+        ConcurrentMap<String, YangSourceFromDependency> byContent = Maps.newConcurrentMap();
+
+        for (YangSourceFromDependency yangFromDependency : list) {
+            try (InputStream dataStream = yangFromDependency.openStream()) {
+                String contents = IOUtils.toString(dataStream);
+                byContent.putIfAbsent(contents, yangFromDependency);
+            }
+
+        }
+        List<InputStream> inputs = new ArrayList<>(byContent.size());
+        for (YangSourceFromDependency entry : byContent.values()) {
+            inputs.add(entry.openStream());
+        }
+        return inputs;
+    }
+
     static class YangProvider {
 
         void addYangsToMetaInf(Log log, MavenProject project, File yangFilesRootDir, File[] excludedFiles)