Capture input files in YangToSourcesState

We need to hash the files we use for input into the
EffectiveModelContext, so we can compare them when we next run.

This patch does not completely deal with the dependencies, as they are a
tad more complicated.

JIRA: YANGTOOLS-1165
Change-Id: Id2d3154d564918e2ee37d4de71f336863afc7da1
Signed-off-by: Ruslan Kashapov <ruslan.kashapov@pantheon.tech>
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>

diff --git a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/CapturingInputStream.java b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/CapturingInputStream.java
new file mode 100644 (file)
index 0000000..cc30840
--- /dev/null
+++ b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/CapturingInputStream.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2023 PANTHEON.tech, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang2sources.plugin;
+
+import com.google.common.hash.Hashing;
+import com.google.common.hash.HashingInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An {@link InputStream} which captures the sum of its contents.
+ */
+final class CapturingInputStream extends FilterInputStream {
+    private long size;
+
+    CapturingInputStream(final InputStream in) {
+        super(new HashingInputStream(Hashing.crc32c(), in));
+    }
+
+    @Override
+    public int read() throws IOException {
+        final int ret = super.read();
+        if (ret != -1) {
+            size++;
+        }
+        return ret;
+    }
+
+    @Override
+    @SuppressWarnings("checkstyle:parameterName")
+    public int read(final byte[] b, final int off, final int len) throws IOException {
+        final int ret = super.read(b, off, len);
+        if (ret != -1) {
+            size += ret;
+        }
+        return ret;
+    }
+
+    long size() {
+        return size;
+    }
+
+    int crc32c() {
+        return ((HashingInputStream) in).hash().asInt();
+    }
+}
\ No newline at end of file

diff --git a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/FileState.java b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/FileState.java
index e06e65e0af4e1b809c2c60a761300e92cc4cea24..e6428038ccf67c6b718a00a22b3cd04d49641957 100644 (file)
--- a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/FileState.java
+++ b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/FileState.java
@@ -38,6 +38,13 @@ record FileState(@NonNull String path, long size, int crc32) {
         requireNonNull(path);
     }
 
+    static @NonNull FileState ofFile(final File file) throws IOException {
+        try (var cis = new CapturingInputStream(Files.newInputStream(file.toPath()))) {
+            cis.readAllBytes();
+            return new FileState(file.getPath(), cis.size(), cis.crc32c());
+        }
+    }
+
     static @NonNull FileState ofWrittenFile(final File file, final FileContent content) throws IOException {
         try (var out = new CapturingOutputStream(Files.newOutputStream(file.toPath()))) {
             content.writeTo(out);

diff --git a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java
index 91da39a7aaa25f01a20d69a8d4158f0d1fc8f53b..e43271d4efeacfe89e9b21b9500ded9bc46430a7 100644 (file)
--- a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java
+++ b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java
@@ -31,6 +31,7 @@ import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.maven.plugin.MojoExecutionException;
@@ -112,7 +113,7 @@ class YangToSourcesProcessor {
     }
 
     void execute() throws MojoExecutionException, MojoFailureException {
-        YangToSourcesState prevState;
+        final YangToSourcesState prevState;
         try {
             prevState = stateStorage.loadState();
         } catch (IOException e) {
@@ -169,6 +170,7 @@ class YangToSourcesProcessor {
          * Check if any of the listed files changed. If no changes occurred, simply return empty, which indicates
          * end of execution.
          */
+        // FIXME: YANGTOOLS-745: remove this check FileStates instead
         if (!Stream.concat(yangFilesInProject.stream(), dependencies.stream().map(ScannedDependency::file))
                 .anyMatch(buildContext::hasDelta)) {
             LOG.info("{} None of {} input files changed", LOG_PREFIX, yangFilesInProject.size() + dependencies.size());
@@ -176,6 +178,17 @@ class YangToSourcesProcessor {
         }
 
         final Stopwatch watch = Stopwatch.createStarted();
+        // Determine hash/size of YANG input files in parallel
+        final var projectYangs = new FileStateSet(yangFilesInProject.parallelStream()
+            .map(file -> {
+                try {
+                    return FileState.ofFile(file);
+                } catch (IOException e) {
+                    throw new IllegalStateException("Failed to read " + file, e);
+                }
+            })
+            .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
+
         final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
             .map(file -> {
                 final YangTextSchemaSource textSource = YangTextSchemaSource.forPath(file.toPath());
@@ -189,6 +202,20 @@ class YangToSourcesProcessor {
         LOG.debug("Found project files: {}", yangFilesInProject);
         LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
 
+        // Determine hash/size of dependency files
+        // TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
+        //       not the entire file
+        final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
+            .map(ScannedDependency::file)
+            .map(file -> {
+                try {
+                    return FileState.ofFile(file);
+                } catch (IOException e) {
+                    throw new IllegalStateException("Failed to read " + file, e);
+                }
+            })
+            .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
+
         final var outputFiles = ImmutableList.<FileState>builder();
         boolean sourcesPersisted = false;
 
@@ -242,11 +269,10 @@ class YangToSourcesProcessor {
             }
         }
 
-        // FIXME: store these files into state, so that we can verify/clean up
         final var outputState = new YangToSourcesState(
             codeGenerators.stream()
                 .collect(ImmutableMap.toImmutableMap(GeneratorTaskFactory::getIdentifier, GeneratorTaskFactory::arg)),
-            new FileStateSet(ImmutableMap.copyOf(uniqueOutputFiles)));
+            projectYangs, dependencyYangs, new FileStateSet(ImmutableMap.copyOf(uniqueOutputFiles)));
 
         try {
             stateStorage.storeState(outputState);

diff --git a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesState.java b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesState.java
index 934c62e8238831b48692cc0af27754b7176d1650..7750a07a57d9d237dc93cb0f71216b730f075b3b 100644 (file)
--- a/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesState.java
+++ b/plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesState.java
@@ -20,16 +20,18 @@ import org.opendaylight.yangtools.concepts.WritableObject;
 /**
  * State of the result of a {@link YangToSourcesMojo} execution run.
  */
-// FIXME: expand to capture:
-//        - input YANG files
 record YangToSourcesState(
         @NonNull ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs,
+        @NonNull FileStateSet projectYangs,
+        @NonNull FileStateSet dependencyYangs,
         @NonNull FileStateSet outputFiles) implements WritableObject {
     // 'ymp' followed by a byte value '1'
     private static final int MAGIC = 0x796D7001;
 
     YangToSourcesState {
         requireNonNull(fileGeneratorArgs);
+        requireNonNull(projectYangs);
+        requireNonNull(dependencyYangs);
         requireNonNull(outputFiles);
     }
 
@@ -38,13 +40,16 @@ record YangToSourcesState(
         if (magic != MAGIC) {
             throw new IOException("Unrecognized magic " + Integer.toHexString(magic));
         }
-        return new YangToSourcesState(readConfigurations(in), FileStateSet.readFrom(in));
+        return new YangToSourcesState(readConfigurations(in),
+            FileStateSet.readFrom(in), FileStateSet.readFrom(in), FileStateSet.readFrom(in));
     }
 
     @Override
     public void writeTo(final DataOutput out) throws IOException {
         out.writeInt(MAGIC);
         writeConfigurations(out, fileGeneratorArgs.values());
+        projectYangs.writeTo(out);
+        dependencyYangs.writeTo(out);
         outputFiles.writeTo(out);
     }