import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.project.MavenProject;
import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorException;
import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorFactory;
import org.opendaylight.yangtools.yang.common.YangConstants;
-import org.opendaylight.yangtools.yang.model.repo.api.YangIRSchemaSource;
-import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
+import org.opendaylight.yangtools.yang.model.spi.source.DelegatedYangTextSource;
+import org.opendaylight.yangtools.yang.model.spi.source.YangIRSchemaSource;
+import org.opendaylight.yangtools.yang.model.spi.source.YangTextSource;
import org.opendaylight.yangtools.yang.parser.api.YangParserConfiguration;
import org.opendaylight.yangtools.yang.parser.api.YangParserException;
import org.opendaylight.yangtools.yang.parser.api.YangParserFactory;
import org.sonatype.plexus.build.incremental.BuildContext;
import org.sonatype.plexus.build.incremental.DefaultBuildContext;
+// FIXME: rename to Execution
+// FIXME: final
class YangToSourcesProcessor {
private static final Logger LOG = LoggerFactory.getLogger(YangToSourcesProcessor.class);
private static final YangParserFactory DEFAULT_PARSER_FACTORY;
final var stateListBuilder = ImmutableList.<FileState>builderWithExpectedSize(modelsInProject.size());
for (var source : modelsInProject) {
- final File file = new File(withMetaInf, source.getIdentifier().toYangFilename());
- stateListBuilder.add(FileState.ofWrittenFile(file, source::copyTo));
- LOG.debug("Created file {} for {}", file, source.getIdentifier());
+ final File file = new File(withMetaInf, source.sourceId().toYangFilename());
+ stateListBuilder.add(FileState.ofWrittenFile(file,
+ out -> source.asByteSource(StandardCharsets.UTF_8).copyTo(out)));
+ LOG.debug("Created file {} for {}", file, source.sourceId());
}
ProjectFileAccess.addResourceDir(project, generatedYangDir);
private final ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs;
private final @NonNull MavenProject project;
private final boolean inspectDependencies;
- private final BuildContext buildContext;
+ private final @NonNull BuildContext buildContext;
private final YangProvider yangProvider;
private final StateStorage stateStorage;
+ private final String projectBuildDirectory;
private YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGeneratorsArgs,
this.project = requireNonNull(project);
this.inspectDependencies = inspectDependencies;
this.yangProvider = requireNonNull(yangProvider);
- stateStorage = StateStorage.of(buildContext, stateFilePath(project.getBuild().getDirectory()));
+ projectBuildDirectory = project.getBuild().getDirectory();
+ stateStorage = StateStorage.of(buildContext, stateFilePath(projectBuildDirectory));
parserFactory = DEFAULT_PARSER_FACTORY;
}
}
void execute() throws MojoExecutionException, MojoFailureException {
- final YangToSourcesState prevState;
+ YangToSourcesState prevState;
try {
prevState = stateStorage.loadState();
} catch (IOException e) {
}
if (prevState == null) {
LOG.debug("{} no previous execution state present", LOG_PREFIX);
+ prevState = new YangToSourcesState(ImmutableMap.of(),
+ FileStateSet.empty(), FileStateSet.empty(), FileStateSet.empty());
}
// Collect all files in the current project.
}
// We need to instantiate all code generators to determine required import resolution mode
- final List<GeneratorTask> codeGenerators = instantiateGenerators();
+ final var codeGenerators = instantiateGenerators();
if (codeGenerators.isEmpty()) {
LOG.warn("{} No code generators provided", LOG_PREFIX);
wipeAllState(prevState);
return;
}
- final Set<YangParserConfiguration> parserConfigs = codeGenerators.stream()
- .map(GeneratorTask::parserConfig)
- .collect(Collectors.toUnmodifiableSet());
-
LOG.info("{} Inspecting {}", LOG_PREFIX, yangFilesRootDir);
// All files which affect YANG context. This minimally includes all files in the current project, but optionally
dependencies = List.of();
}
- /*
- * Check if any of the listed files changed. If no changes occurred, simply return empty, which indicates
- * end of execution.
- */
- // FIXME: YANGTOOLS-745: remove this check FileStates instead
- if (!Stream.concat(yangFilesInProject.stream(), dependencies.stream().map(ScannedDependency::file))
- .anyMatch(buildContext::hasDelta)) {
- LOG.info("{} None of {} input files changed", LOG_PREFIX, yangFilesInProject.size() + dependencies.size());
- return;
- }
-
- final Stopwatch watch = Stopwatch.createStarted();
- // Determine hash/size of YANG input files in parallel
+ // Determine hash/size of YANG input files and dependencies in parallel
+ final var hashTimer = Stopwatch.createStarted();
final var projectYangs = new FileStateSet(yangFilesInProject.parallelStream()
.map(file -> {
try {
}
})
.collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
-
- final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
- .map(file -> {
- final YangTextSchemaSource textSource = YangTextSchemaSource.forPath(file.toPath());
- try {
- return Map.entry(textSource, TextToIRTransformer.transformText(textSource));
- } catch (YangSyntaxErrorException | IOException e) {
- throw new IllegalArgumentException("Failed to parse " + file, e);
- }
- })
- .collect(Collectors.toList());
- LOG.debug("Found project files: {}", yangFilesInProject);
- LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
-
- // Determine hash/size of dependency files
// TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
// not the entire file
final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
}
})
.collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
+ LOG.debug("{} Input state determined in {}", LOG_PREFIX, hashTimer);
- final var outputFiles = ImmutableList.<FileState>builder();
- Collection<YangTextSchemaSource> modelsInProject = null;
+ // We have collected our current inputs and previous state. Instantiate a support object which will guide us for
+ // the rest of the way.
+ final var buildSupport = new IncrementalBuildSupport(prevState,
+ codeGenerators.stream()
+ .collect(ImmutableMap.toImmutableMap(GeneratorTask::getIdentifier, GeneratorTask::arg)),
+ projectYangs, dependencyYangs);
+
+ // Check if any inputs changed, which is supposed to be fast. If they did not, we need to also validate our
+ // our previous are also up-to-date.
+ if (!buildSupport.inputsChanged()) {
+ final boolean outputsChanged;
+ try {
+ outputsChanged = buildSupport.outputsChanged(projectBuildDirectory);
+ } catch (IOException e) {
+ throw new MojoFailureException("Failed to reconcile generation outputs", e);
+ }
+
+ if (!outputsChanged) {
+ // FIXME: YANGTOOLS-745: still need to add all resources/directories to maven project
+ LOG.info("{}: Everything is up to date, nothing to do", LOG_PREFIX);
+ return;
+ }
+ }
- for (YangParserConfiguration parserConfig : parserConfigs) {
+ final Stopwatch watch = Stopwatch.createStarted();
+
+ final List<Entry<YangTextSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
+ .map(file -> {
+ final var textSource = YangTextSource.forPath(file.toPath());
+ try {
+ return Map.entry(textSource, TextToIRTransformer.transformText(textSource));
+ } catch (YangSyntaxErrorException | IOException e) {
+ throw new IllegalArgumentException("Failed to parse " + file, e);
+ }
+ })
+ .collect(Collectors.toList());
+ LOG.debug("Found project files: {}", yangFilesInProject);
+ LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
+
+ final var outputFiles = ImmutableList.<FileState>builder();
+ Collection<YangTextSource> modelsInProject = null;
+ for (var parserConfig : codeGenerators.stream().map(GeneratorTask::parserConfig).collect(Collectors.toSet())) {
final var moduleReactor = createReactor(yangFilesInProject, parserConfig, dependencies, parsed);
final var yangSw = Stopwatch.createStarted();
final List<FileState> files;
try {
files = factory.execute(project, buildContext, holder);
- } catch (FileGeneratorException | IOException e) {
+ } catch (FileGeneratorException e) {
throw new MojoFailureException(LOG_PREFIX + " Generator " + factory + " failed", e);
}
}
// add META_INF/services
- File generatedServicesDir = new File(new File(project.getBuild().getDirectory(), "generated-sources"), "spi");
+ File generatedServicesDir = new File(new File(projectBuildDirectory, "generated-sources"), "spi");
ProjectFileAccess.addResourceDir(project, generatedServicesDir);
LOG.debug("{} Yang services files from: {} marked as resources: {}", LOG_PREFIX, generatedServicesDir,
META_INF_YANG_SERVICES_STRING_JAR);
}
}
- final var outputState = new YangToSourcesState(
- codeGenerators.stream()
- .collect(ImmutableMap.toImmutableMap(GeneratorTask::getIdentifier, GeneratorTask::arg)),
- projectYangs, dependencyYangs, new FileStateSet(ImmutableMap.copyOf(uniqueOutputFiles)));
+ // Reconcile push output files into project directory and acquire the execution state
+ final YangToSourcesState outputState;
+ try {
+ outputState = buildSupport.reconcileOutputFiles(buildContext, projectBuildDirectory, uniqueOutputFiles);
+ } catch (IOException e) {
+ throw new MojoFailureException("Failed to reconcile output files", e);
+ }
+ // Store execution state
try {
stateStorage.storeState(outputState);
} catch (IOException e) {
}
private void wipeAllState(final YangToSourcesState prevState) throws MojoExecutionException {
- if (prevState != null) {
- for (var file : prevState.outputFiles().fileStates().keySet()) {
- try {
- Files.deleteIfExists(Path.of(file));
- } catch (IOException e) {
- throw new MojoExecutionException("Failed to remove file " + file, e);
- }
- }
+ try {
+ prevState.deleteOutputFiles();
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed to delete output files", e);
}
-
try {
stateStorage.deleteState();
} catch (IOException e) {
}
}
- private List<GeneratorTask> instantiateGenerators() throws MojoExecutionException {
+ private ImmutableList<@NonNull GeneratorTask> instantiateGenerators() throws MojoExecutionException {
// Search for available FileGenerator implementations
- final Map<String, FileGeneratorFactory> factories = Maps.uniqueIndex(
+ final var factories = Maps.uniqueIndex(
ServiceLoader.load(FileGeneratorFactory.class), FileGeneratorFactory::getIdentifier);
// FIXME: iterate over fileGeneratorArg instances (configuration), not factories (environment)
// Assign instantiate FileGenerators with appropriate configuration
- final var generators = new ArrayList<GeneratorTask>(factories.size());
- for (Entry<String, FileGeneratorFactory> entry : factories.entrySet()) {
- final String id = entry.getKey();
- FileGeneratorArg arg = fileGeneratorArgs.get(id);
+ final var builder = ImmutableList.<@NonNull GeneratorTask>builderWithExpectedSize(factories.size());
+ for (var entry : factories.entrySet()) {
+ final var id = entry.getKey();
+ var arg = fileGeneratorArgs.get(id);
if (arg == null) {
LOG.debug("{} No configuration for {}, using empty", LOG_PREFIX, id);
arg = new FileGeneratorArg(id);
} catch (FileGeneratorException e) {
throw new MojoExecutionException("File generator " + id + " failed", e);
}
- generators.add(task);
+ builder.add(task);
LOG.info("{} Code generator {} instantiated", LOG_PREFIX, id);
}
}
);
- return generators;
+ return builder.build();
}
@SuppressWarnings("checkstyle:illegalCatch")
private @NonNull ProcessorModuleReactor createReactor(final List<File> yangFilesInProject,
final YangParserConfiguration parserConfig, final Collection<ScannedDependency> dependencies,
- final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed) throws MojoExecutionException {
+ final List<Entry<YangTextSource, YangIRSchemaSource>> parsed) throws MojoExecutionException {
try {
- final var sourcesInProject = new ArrayList<YangTextSchemaSource>(yangFilesInProject.size());
+ final var sourcesInProject = new ArrayList<YangTextSource>(yangFilesInProject.size());
final var parser = parserFactory.createParser(parserConfig);
for (var entry : parsed) {
final var textSource = entry.getKey();
final var astSource = entry.getValue();
parser.addSource(astSource);
- if (!astSource.getIdentifier().equals(textSource.getIdentifier())) {
+ if (!astSource.sourceId().equals(textSource.sourceId())) {
// AST indicates a different source identifier, make sure we use that
- sourcesInProject.add(YangTextSchemaSource.delegateForByteSource(astSource.getIdentifier(),
- textSource));
+ sourcesInProject.add(new DelegatedYangTextSource(astSource.sourceId(), textSource));
} else {
sourcesInProject.add(textSource);
}
Code Review / yangtools.git / blobdiff