import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import org.eclipse.jdt.annotation.NonNull;
}
static @NonNull FileState ofFile(final File file) throws IOException {
- try (var cis = new CapturingInputStream(Files.newInputStream(file.toPath()))) {
+ return ofFile(file.toPath());
+ }
+
+ static @NonNull FileState ofFile(final Path file) throws IOException {
+ try (var cis = new CapturingInputStream(Files.newInputStream(file))) {
cis.readAllBytes();
- return new FileState(file.getPath(), cis.size(), cis.crc32c());
+ return new FileState(file.toString(), cis.size(), cis.crc32c());
}
}
--- /dev/null
+/*
+ * Copyright (c) 2023 PANTHEON.tech, s.r.o. and others. All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.yangtools.yang2sources.plugin;
+
+import static java.util.Objects.requireNonNull;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jdt.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.plexus.build.incremental.BuildContext;
+
+/**
+ * Support for incremental builds. It provides interface between {@link YangToSourcesProcessor} execution, observed
+ * project input state (like dependencies, project YANG files, plugins and their configuration, etc.) and the contents
+ * of the project build directory.
+ *
+ * <p>
+ * While the logic here could be integrated, we keep it separate so we can test it separately.
+ */
+// FIXME: rename to ExecutionSupport or something similar
+final class IncrementalBuildSupport {
+ private static final Logger LOG = LoggerFactory.getLogger(IncrementalBuildSupport.class);
+ private static final List<String> TRANSIENT_DIRECTORIES = List.of("generated-resources", "generated-sources");
+
+ private final @NonNull ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs;
+ private final @NonNull FileStateSet projectYangs;
+ private final @NonNull FileStateSet dependencyYangs;
+
+ private final @Nullable YangToSourcesState previousState;
+
+ IncrementalBuildSupport(final YangToSourcesState previousState,
+ final ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs, final FileStateSet projectYangs,
+ final FileStateSet dependencyYangs) {
+ this.previousState = previousState;
+ this.fileGeneratorArgs = requireNonNull(fileGeneratorArgs);
+ this.projectYangs = requireNonNull(projectYangs);
+ this.dependencyYangs = requireNonNull(dependencyYangs);
+ }
+
+ /**
+ * Compare previous state against current inputs and return {@core true} if any of the parameters changed.
+ *
+ * @return {@code true} if restored state is different from currently-observed state
+ */
+ boolean inputsChanged() {
+ // Local variable to aid null analysis
+ final var local = previousState;
+ if (local == null) {
+ LOG.debug("{}: no previous input state", YangToSourcesProcessor.LOG_PREFIX);
+ return true;
+ }
+ if (!fileGeneratorArgs.equals(local.fileGeneratorArgs())) {
+ LOG.debug("{}: file generator arguments changed from {} to {}", YangToSourcesProcessor.LOG_PREFIX,
+ fileGeneratorArgs, local.fileGeneratorArgs());
+ return true;
+ }
+ if (!projectYangs.equals(local.projectYangs())) {
+ LOG.debug("{}: project YANG files changed", YangToSourcesProcessor.LOG_PREFIX);
+ return true;
+ }
+ if (!dependencyYangs.equals(local.dependencyYangs())) {
+ LOG.debug("{}: dependency YANG files changed", YangToSourcesProcessor.LOG_PREFIX);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Compare previous state to the contents of the specified project build directory. This method detects any changes
+ * to existence/contents of files recorded in previous state as well as any extra files created in per-generator
+ * sub-directories of the project build directory.
+ *
+ * @param projectBuildDirectory Current project build directory
+ * @return {@code true} if restored state and the build directory differ
+ */
+ boolean outputsChanged(final String projectBuildDirectory) throws IOException {
+ // Local variable to aid null analysis
+ final var local = previousState;
+ if (local == null) {
+ LOG.debug("{}: no previous output state", YangToSourcesProcessor.LOG_PREFIX);
+ return true;
+ }
+
+ // Compare explicit mentions first
+ final var outputFiles = local.outputFiles().fileStates();
+ if (outputsChanged(outputFiles.values())) {
+ return true;
+ }
+
+ // Make sure all discovered files for codegen plugins are accounted for
+ for (var pluginName : fileGeneratorArgs.keySet()) {
+ for (var directory : TRANSIENT_DIRECTORIES) {
+ final var dirPath = pluginSubdirectory(projectBuildDirectory, pluginName, directory);
+ if (Files.isDirectory(dirPath)) {
+ final var mismatch = Files.walk(dirPath)
+ .filter(Files::isRegularFile)
+ .map(Path::toString)
+ .allMatch(path -> {
+ if (outputFiles.containsKey(path)) {
+ return true;
+ }
+ LOG.info("{}: unexpected output file {}", YangToSourcesProcessor.LOG_PREFIX, path);
+ return false;
+ });
+ if (!mismatch) {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+ }
+
+ private static boolean outputsChanged(final Collection<FileState> outputFiles) throws IOException {
+ for (var prev : outputFiles) {
+ final var current = FileState.ofFile(Path.of(prev.path()));
+ if (!prev.equals(current)) {
+ LOG.debug("{}: output file changed from {} to {}", YangToSourcesProcessor.LOG_PREFIX, prev, current);
+ LOG.info("{}: output file {} changed", YangToSourcesProcessor.LOG_PREFIX, prev.path());
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @NonNull YangToSourcesState reconcileOutputFiles(final BuildContext buildContext,
+ final String projectBuildDirectory, final Map<String, FileState> outputFiles) throws IOException {
+ final var local = previousState;
+ if (local != null) {
+ // Delete any file mentioned in previous state and not output by us
+ for (var file : local.outputFiles().fileStates().keySet()) {
+ if (!outputFiles.containsKey(file)) {
+ final var path = Path.of(file);
+ if (Files.deleteIfExists(path)) {
+ buildContext.refresh(path.toFile());
+ }
+ }
+ }
+
+ // Recursively delete all plugin directories configured
+ for (var plugin : local.fileGeneratorArgs().keySet()) {
+ if (!fileGeneratorArgs.containsKey(plugin)) {
+ for (var directory : TRANSIENT_DIRECTORIES) {
+ deleteRecursively(buildContext, pluginSubdirectory(projectBuildDirectory, plugin, directory));
+ }
+ }
+ }
+ }
+
+ // Now examine each plugin's output and remove any file not mentioned in outputs
+ for (var plugin : fileGeneratorArgs.keySet()) {
+ for (var directory : TRANSIENT_DIRECTORIES) {
+ final var dirPath = pluginSubdirectory(projectBuildDirectory, plugin, directory);
+ if (Files.isDirectory(dirPath)) {
+ Files.walk(dirPath)
+ .filter(Files::isRegularFile)
+ .filter(path -> !outputFiles.containsKey(path.toString()))
+ .map(Path::toFile)
+ .forEach(file -> {
+ if (file.delete()) {
+ // Notify BuildContext of the deleted file
+ buildContext.refresh(file);
+ }
+ });
+ }
+ }
+ }
+
+ return new YangToSourcesState(fileGeneratorArgs, projectYangs, dependencyYangs,
+ new FileStateSet(ImmutableMap.copyOf(outputFiles)));
+ }
+
+ static Path pluginSubdirectory(final String projectBuildDirectory, final String pluginName, final String subdir) {
+ return Path.of(projectBuildDirectory, subdir, pluginName);
+ }
+
+ private static void deleteRecursively(final BuildContext buildContext, final Path path) throws IOException {
+ if (Files.isDirectory(path)) {
+ Files.walk(path).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(file -> {
+ if (file.delete()) {
+ // Notify BuildContext of the deleted file
+ buildContext.refresh(file);
+ }
+ });
+ }
+ }
+}
/**
* An incremental state reactor. Allows resolution of a SchemaContext based on a set of sources.
- *
- * @author Robert Varga
*/
+// FIXME: reneame to ExecutionModuleReactor or similar
final class ProcessorModuleReactor {
private static final Logger LOG = LoggerFactory.getLogger(ProcessorModuleReactor.class);
}
private @NonNull File buildDirectoryFor(final String name) {
- return new File(project.getBuild().getDirectory()
- + File.separatorChar + name
- + File.separatorChar + buildDirSuffix);
+ return IncrementalBuildSupport.pluginSubdirectory(project.getBuild().getDirectory(), buildDirSuffix, name)
+ .toFile();
}
}
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.project.MavenProject;
import org.eclipse.jdt.annotation.NonNull;
+import org.eclipse.jdt.annotation.Nullable;
import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorException;
import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorFactory;
import org.opendaylight.yangtools.yang.common.YangConstants;
import org.sonatype.plexus.build.incremental.BuildContext;
import org.sonatype.plexus.build.incremental.DefaultBuildContext;
+// FIXME: rename to Execution
+// FIXME: final
class YangToSourcesProcessor {
private static final Logger LOG = LoggerFactory.getLogger(YangToSourcesProcessor.class);
private static final YangParserFactory DEFAULT_PARSER_FACTORY;
private final ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs;
private final @NonNull MavenProject project;
private final boolean inspectDependencies;
- private final BuildContext buildContext;
+ private final @NonNull BuildContext buildContext;
private final YangProvider yangProvider;
private final StateStorage stateStorage;
+ private final String projectBuildDirectory;
private YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGeneratorsArgs,
this.project = requireNonNull(project);
this.inspectDependencies = inspectDependencies;
this.yangProvider = requireNonNull(yangProvider);
- stateStorage = StateStorage.of(buildContext, stateFilePath(project.getBuild().getDirectory()));
+ projectBuildDirectory = project.getBuild().getDirectory();
+ stateStorage = StateStorage.of(buildContext, stateFilePath(projectBuildDirectory));
parserFactory = DEFAULT_PARSER_FACTORY;
}
}
// We need to instantiate all code generators to determine required import resolution mode
- final List<GeneratorTask> codeGenerators = instantiateGenerators();
+ final var codeGenerators = instantiateGenerators();
if (codeGenerators.isEmpty()) {
LOG.warn("{} No code generators provided", LOG_PREFIX);
wipeAllState(prevState);
return;
}
- final Set<YangParserConfiguration> parserConfigs = codeGenerators.stream()
- .map(GeneratorTask::parserConfig)
- .collect(Collectors.toUnmodifiableSet());
-
LOG.info("{} Inspecting {}", LOG_PREFIX, yangFilesRootDir);
// All files which affect YANG context. This minimally includes all files in the current project, but optionally
dependencies = List.of();
}
- /*
- * Check if any of the listed files changed. If no changes occurred, simply return empty, which indicates
- * end of execution.
- */
- // FIXME: YANGTOOLS-745: remove this check FileStates instead
- if (!Stream.concat(yangFilesInProject.stream(), dependencies.stream().map(ScannedDependency::file))
- .anyMatch(buildContext::hasDelta)) {
- LOG.info("{} None of {} input files changed", LOG_PREFIX, yangFilesInProject.size() + dependencies.size());
- return;
- }
-
- final Stopwatch watch = Stopwatch.createStarted();
- // Determine hash/size of YANG input files in parallel
+ // Determine hash/size of YANG input files and dependencies in parallel
+ final var hashTimer = Stopwatch.createStarted();
final var projectYangs = new FileStateSet(yangFilesInProject.parallelStream()
.map(file -> {
try {
}
})
.collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
+ // TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
+ // not the entire file
+ final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
+ .map(ScannedDependency::file)
+ .map(file -> {
+ try {
+ return FileState.ofFile(file);
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to read " + file, e);
+ }
+ })
+ .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
+ LOG.debug("{} Input state determined in {}", LOG_PREFIX, hashTimer);
+
+ // We have collected our current inputs and previous state. Instantiate a support object which will guide us for
+ // the rest of the way.
+ final var buildSupport = new IncrementalBuildSupport(prevState,
+ codeGenerators.stream()
+ .collect(ImmutableMap.toImmutableMap(GeneratorTask::getIdentifier, GeneratorTask::arg)),
+ projectYangs, dependencyYangs);
+
+ // Check if any inputs changed, which is supposed to be fast. If they did not, we need to also validate our
+ // our previous are also up-to-date.
+ if (!buildSupport.inputsChanged()) {
+ final boolean outputsChanged;
+ try {
+ outputsChanged = buildSupport.outputsChanged(projectBuildDirectory);
+ } catch (IOException e) {
+ throw new MojoFailureException("Failed to reconcile generation outputs", e);
+ }
+
+ if (!outputsChanged) {
+ // FIXME: YANGTOOLS-745: still need to add all resources/directories to maven project
+ LOG.info("{}: Everything is up to date, nothing to do", LOG_PREFIX);
+ return;
+ }
+ }
+
+ final Stopwatch watch = Stopwatch.createStarted();
final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
.map(file -> {
LOG.debug("Found project files: {}", yangFilesInProject);
LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
- // Determine hash/size of dependency files
- // TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
- // not the entire file
- final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
- .map(ScannedDependency::file)
- .map(file -> {
- try {
- return FileState.ofFile(file);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to read " + file, e);
- }
- })
- .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
-
final var outputFiles = ImmutableList.<FileState>builder();
Collection<YangTextSchemaSource> modelsInProject = null;
-
- for (YangParserConfiguration parserConfig : parserConfigs) {
+ for (var parserConfig : codeGenerators.stream().map(GeneratorTask::parserConfig).collect(Collectors.toSet())) {
final var moduleReactor = createReactor(yangFilesInProject, parserConfig, dependencies, parsed);
final var yangSw = Stopwatch.createStarted();
}
// add META_INF/services
- File generatedServicesDir = new File(new File(project.getBuild().getDirectory(), "generated-sources"), "spi");
+ File generatedServicesDir = new File(new File(projectBuildDirectory, "generated-sources"), "spi");
ProjectFileAccess.addResourceDir(project, generatedServicesDir);
LOG.debug("{} Yang services files from: {} marked as resources: {}", LOG_PREFIX, generatedServicesDir,
META_INF_YANG_SERVICES_STRING_JAR);
}
}
- final var outputState = new YangToSourcesState(
- codeGenerators.stream()
- .collect(ImmutableMap.toImmutableMap(GeneratorTask::getIdentifier, GeneratorTask::arg)),
- projectYangs, dependencyYangs, new FileStateSet(ImmutableMap.copyOf(uniqueOutputFiles)));
+ // Reconcile push output files into project directory and acquire the execution state
+ final YangToSourcesState outputState;
+ try {
+ outputState = buildSupport.reconcileOutputFiles(buildContext, projectBuildDirectory, uniqueOutputFiles);
+ } catch (IOException e) {
+ throw new MojoFailureException("Failed to reconcile output files", e);
+ }
+ // Store execution state
try {
stateStorage.storeState(outputState);
} catch (IOException e) {
}
}
- private void wipeAllState(final YangToSourcesState prevState) throws MojoExecutionException {
+ private void wipeAllState(final @Nullable YangToSourcesState prevState) throws MojoExecutionException {
if (prevState != null) {
for (var file : prevState.outputFiles().fileStates().keySet()) {
try {
/**
* State of the result of a {@link YangToSourcesMojo} execution run.
*/
+// FIXME: rename to ExecutionState
record YangToSourcesState(
@NonNull ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs,
@NonNull FileStateSet projectYangs,
FileStateSet.readFrom(in), FileStateSet.readFrom(in), FileStateSet.readFrom(in));
}
+ static @NonNull YangToSourcesState empty() {
+ return new YangToSourcesState(ImmutableMap.of(),
+ FileStateSet.empty(), FileStateSet.empty(), FileStateSet.empty());
+ }
+
@Override
public void writeTo(final DataOutput out) throws IOException {
out.writeInt(MAGIC);
Code Review / yangtools.git / commitdiff