*/
package org.opendaylight.yangtools.yang2sources.plugin;
-import static com.google.common.base.Preconditions.checkState;
import static java.util.Objects.requireNonNull;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Optional;
+import java.util.NoSuchElementException;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.project.MavenProject;
import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorException;
import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorFactory;
import org.opendaylight.yangtools.yang.common.YangConstants;
-import org.opendaylight.yangtools.yang.model.repo.api.YangIRSchemaSource;
-import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
-import org.opendaylight.yangtools.yang.parser.api.YangParser;
+import org.opendaylight.yangtools.yang.model.spi.source.DelegatedYangTextSource;
+import org.opendaylight.yangtools.yang.model.spi.source.YangIRSchemaSource;
+import org.opendaylight.yangtools.yang.model.spi.source.YangTextSource;
import org.opendaylight.yangtools.yang.parser.api.YangParserConfiguration;
import org.opendaylight.yangtools.yang.parser.api.YangParserException;
import org.opendaylight.yangtools.yang.parser.api.YangParserFactory;
import org.sonatype.plexus.build.incremental.BuildContext;
import org.sonatype.plexus.build.incremental.DefaultBuildContext;
+// FIXME: rename to Execution
+// FIXME: final
class YangToSourcesProcessor {
private static final Logger LOG = LoggerFactory.getLogger(YangToSourcesProcessor.class);
private static final YangParserFactory DEFAULT_PARSER_FACTORY;
static {
- final Iterator<YangParserFactory> it = ServiceLoader.load(YangParserFactory.class).iterator();
- checkState(it.hasNext(), "Failed to find a YangParserFactory implementation");
- DEFAULT_PARSER_FACTORY = it.next();
+ try {
+ DEFAULT_PARSER_FACTORY = ServiceLoader.load(YangParserFactory.class).iterator().next();
+ } catch (NoSuchElementException e) {
+ throw new IllegalStateException("Failed to find a YangParserFactory implementation", e);
+ }
}
- static final String LOG_PREFIX = "yang-to-sources:";
private static final String META_INF_STR = "META-INF";
private static final String YANG_STR = "yang";
+ static final String LOG_PREFIX = "yang-to-sources:";
static final String META_INF_YANG_STRING = META_INF_STR + File.separator + YANG_STR;
static final String META_INF_YANG_STRING_JAR = META_INF_STR + "/" + YANG_STR;
static final String META_INF_YANG_SERVICES_STRING_JAR = META_INF_STR + "/" + "services";
+ private static final YangProvider YANG_PROVIDER = (project, modelsInProject) -> {
+ final var generatedYangDir =
+ // FIXME: why are we generating these in "generated-sources"? At the end of the day YANG files are more
+ // resources (except we do not them to be subject to filtering)
+ new File(new File(project.getBuild().getDirectory(), "generated-sources"), "yang");
+ LOG.debug("Generated dir {}", generatedYangDir);
+
+ // copy project's src/main/yang/*.yang to ${project.builddir}/generated-sources/yang/META-INF/yang/
+ // This honors setups like a Eclipse-profile derived one
+ final var withMetaInf = new File(generatedYangDir, YangToSourcesProcessor.META_INF_YANG_STRING);
+ Files.createDirectories(withMetaInf.toPath());
+
+ final var stateListBuilder = ImmutableList.<FileState>builderWithExpectedSize(modelsInProject.size());
+ for (var source : modelsInProject) {
+ final File file = new File(withMetaInf, source.sourceId().toYangFilename());
+ stateListBuilder.add(FileState.ofWrittenFile(file,
+ out -> source.asByteSource(StandardCharsets.UTF_8).copyTo(out)));
+ LOG.debug("Created file {} for {}", file, source.sourceId());
+ }
+
+ ProjectFileAccess.addResourceDir(project, generatedYangDir);
+ LOG.debug("{} YANG files marked as resources: {}", YangToSourcesProcessor.LOG_PREFIX, generatedYangDir);
+
+ return stateListBuilder.build();
+ };
+
private final YangParserFactory parserFactory;
private final File yangFilesRootDir;
private final Set<File> excludedFiles;
private final ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs;
private final @NonNull MavenProject project;
private final boolean inspectDependencies;
- private final BuildContext buildContext;
+ private final @NonNull BuildContext buildContext;
private final YangProvider yangProvider;
private final StateStorage stateStorage;
+ private final String projectBuildDirectory;
private YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGeneratorsArgs,
this.project = requireNonNull(project);
this.inspectDependencies = inspectDependencies;
this.yangProvider = requireNonNull(yangProvider);
- stateStorage = StateStorage.of(buildContext, stateFilePath(project.getBuild().getDirectory()));
+ projectBuildDirectory = project.getBuild().getDirectory();
+ stateStorage = StateStorage.of(buildContext, stateFilePath(projectBuildDirectory));
parserFactory = DEFAULT_PARSER_FACTORY;
}
@VisibleForTesting
- YangToSourcesProcessor(final File yangFilesRootDir, final Collection<File> excludedFiles,
- final List<FileGeneratorArg> fileGenerators, final MavenProject project, final boolean inspectDependencies,
- final YangProvider yangProvider) {
- this(new DefaultBuildContext(), yangFilesRootDir, excludedFiles, ImmutableList.of(),
- project, inspectDependencies, yangProvider);
+ YangToSourcesProcessor(final File yangFilesRootDir, final List<FileGeneratorArg> fileGenerators,
+ final MavenProject project, final YangProvider yangProvider) {
+ this(new DefaultBuildContext(), yangFilesRootDir, List.of(), List.of(), project, false, yangProvider);
}
YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGenerators,
final MavenProject project, final boolean inspectDependencies) {
this(buildContext, yangFilesRootDir, excludedFiles, fileGenerators, project, inspectDependencies,
- YangProvider.getInstance());
+ YANG_PROVIDER);
}
void execute() throws MojoExecutionException, MojoFailureException {
- final YangToSourcesState prevState;
+ YangToSourcesState prevState;
try {
prevState = stateStorage.loadState();
} catch (IOException e) {
}
if (prevState == null) {
LOG.debug("{} no previous execution state present", LOG_PREFIX);
+ prevState = new YangToSourcesState(ImmutableMap.of(),
+ FileStateSet.empty(), FileStateSet.empty(), FileStateSet.empty());
}
// Collect all files in the current project.
if (yangFilesInProject.isEmpty()) {
// No files to process, skip.
LOG.info("{} No input files found", LOG_PREFIX);
+ wipeAllState(prevState);
return;
}
// We need to instantiate all code generators to determine required import resolution mode
- final List<GeneratorTaskFactory> codeGenerators = instantiateGenerators();
+ final var codeGenerators = instantiateGenerators();
if (codeGenerators.isEmpty()) {
LOG.warn("{} No code generators provided", LOG_PREFIX);
+ wipeAllState(prevState);
return;
}
- final Set<YangParserConfiguration> parserConfigs = codeGenerators.stream()
- .map(GeneratorTaskFactory::parserConfig)
- .collect(Collectors.toUnmodifiableSet());
-
LOG.info("{} Inspecting {}", LOG_PREFIX, yangFilesRootDir);
// All files which affect YANG context. This minimally includes all files in the current project, but optionally
}
LOG.info("{} Found {} dependencies in {}", LOG_PREFIX, dependencies.size(), watch);
} else {
- dependencies = ImmutableList.of();
+ dependencies = List.of();
}
- /*
- * Check if any of the listed files changed. If no changes occurred, simply return empty, which indicates
- * end of execution.
- */
- // FIXME: YANGTOOLS-745: remove this check FileStates instead
- if (!Stream.concat(yangFilesInProject.stream(), dependencies.stream().map(ScannedDependency::file))
- .anyMatch(buildContext::hasDelta)) {
- LOG.info("{} None of {} input files changed", LOG_PREFIX, yangFilesInProject.size() + dependencies.size());
- return;
- }
-
- final Stopwatch watch = Stopwatch.createStarted();
- // Determine hash/size of YANG input files in parallel
+ // Determine hash/size of YANG input files and dependencies in parallel
+ final var hashTimer = Stopwatch.createStarted();
final var projectYangs = new FileStateSet(yangFilesInProject.parallelStream()
.map(file -> {
try {
}
})
.collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
-
- final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
- .map(file -> {
- final YangTextSchemaSource textSource = YangTextSchemaSource.forPath(file.toPath());
- try {
- return Map.entry(textSource, TextToIRTransformer.transformText(textSource));
- } catch (YangSyntaxErrorException | IOException e) {
- throw new IllegalArgumentException("Failed to parse " + file, e);
- }
- })
- .collect(Collectors.toList());
- LOG.debug("Found project files: {}", yangFilesInProject);
- LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
-
- // Determine hash/size of dependency files
// TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
// not the entire file
final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
}
})
.collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
+ LOG.debug("{} Input state determined in {}", LOG_PREFIX, hashTimer);
- final var outputFiles = ImmutableList.<FileState>builder();
- boolean sourcesPersisted = false;
+ // We have collected our current inputs and previous state. Instantiate a support object which will guide us for
+ // the rest of the way.
+ final var buildSupport = new IncrementalBuildSupport(prevState,
+ codeGenerators.stream()
+ .collect(ImmutableMap.toImmutableMap(GeneratorTask::getIdentifier, GeneratorTask::arg)),
+ projectYangs, dependencyYangs);
- for (YangParserConfiguration parserConfig : parserConfigs) {
- final Optional<ProcessorModuleReactor> optReactor = createReactor(yangFilesInProject,
- parserConfig, dependencies, parsed);
- if (optReactor.isPresent()) {
- final ProcessorModuleReactor reactor = optReactor.orElseThrow();
+ // Check if any inputs changed, which is supposed to be fast. If they did not, we need to also validate our
+ // our previous are also up-to-date.
+ if (!buildSupport.inputsChanged()) {
+ final boolean outputsChanged;
+ try {
+ outputsChanged = buildSupport.outputsChanged(projectBuildDirectory);
+ } catch (IOException e) {
+ throw new MojoFailureException("Failed to reconcile generation outputs", e);
+ }
+
+ if (!outputsChanged) {
+ // FIXME: YANGTOOLS-745: still need to add all resources/directories to maven project
+ LOG.info("{}: Everything is up to date, nothing to do", LOG_PREFIX);
+ return;
+ }
+ }
- final Stopwatch sw = Stopwatch.createStarted();
- final ContextHolder holder;
+ final Stopwatch watch = Stopwatch.createStarted();
+ final List<Entry<YangTextSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
+ .map(file -> {
+ final var textSource = YangTextSource.forPath(file.toPath());
try {
- holder = reactor.toContext();
- } catch (YangParserException e) {
- throw new MojoFailureException("Failed to process reactor " + reactor, e);
- } catch (IOException e) {
- throw new MojoExecutionException("Failed to read reactor " + reactor, e);
+ return Map.entry(textSource, TextToIRTransformer.transformText(textSource));
+ } catch (YangSyntaxErrorException | IOException e) {
+ throw new IllegalArgumentException("Failed to parse " + file, e);
}
+ })
+ .collect(Collectors.toList());
+ LOG.debug("Found project files: {}", yangFilesInProject);
+ LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
- LOG.info("{} {} YANG models processed in {}", LOG_PREFIX, holder.getContext().getModules().size(), sw);
-
- for (GeneratorTaskFactory factory : codeGenerators) {
- if (!parserConfig.equals(factory.parserConfig())) {
- continue;
- }
-
- final Stopwatch genSw = Stopwatch.createStarted();
- final GeneratorTask task = factory.createTask(project, holder);
- LOG.debug("{} Task {} initialized in {}", LOG_PREFIX, task, genSw);
-
- final List<FileState> files;
- try {
- files = task.execute(buildContext);
- } catch (FileGeneratorException | IOException e) {
- throw new MojoFailureException(LOG_PREFIX + " Generator " + factory + " failed", e);
- }
+ final var outputFiles = ImmutableList.<FileState>builder();
+ Collection<YangTextSource> modelsInProject = null;
+ for (var parserConfig : codeGenerators.stream().map(GeneratorTask::parserConfig).collect(Collectors.toSet())) {
+ final var moduleReactor = createReactor(yangFilesInProject, parserConfig, dependencies, parsed);
+ final var yangSw = Stopwatch.createStarted();
- outputFiles.addAll(files);
+ final ContextHolder holder;
+ try {
+ holder = moduleReactor.toContext();
+ } catch (YangParserException e) {
+ throw new MojoFailureException("Failed to process reactor " + moduleReactor, e);
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed to read reactor " + moduleReactor, e);
+ }
+ LOG.info("{} {} YANG models processed in {}", LOG_PREFIX, holder.getContext().getModules().size(), yangSw);
- LOG.info("{} Sources generated by {}: {} in {}", LOG_PREFIX, factory.generatorName(), files.size(),
- genSw);
+ for (var factory : codeGenerators) {
+ if (!parserConfig.equals(factory.parserConfig())) {
+ continue;
}
- if (!sourcesPersisted) {
- // add META_INF/yang once
- final var models = reactor.getModelsInProject();
- final Collection<FileState> sourceFileStates;
- try {
- sourceFileStates = yangProvider.addYangsToMetaInf(project, models);
- } catch (IOException e) {
- throw new MojoExecutionException("Failed write model files for " + models, e);
- }
-
- sourcesPersisted = true;
- outputFiles.addAll(sourceFileStates);
+ final var genSw = Stopwatch.createStarted();
+ final List<FileState> files;
+ try {
+ files = factory.execute(project, buildContext, holder);
+ } catch (FileGeneratorException e) {
+ throw new MojoFailureException(LOG_PREFIX + " Generator " + factory + " failed", e);
}
+
+ outputFiles.addAll(files);
+ LOG.info("{} Sources generated by {}: {} in {}", LOG_PREFIX, factory.generatorName(), files.size(),
+ genSw);
}
+
+ if (modelsInProject == null) {
+ // FIXME: this is an invariant, we should prepare these separately
+ modelsInProject = moduleReactor.getModelsInProject();
+ }
+ }
+
+ // add META_INF/yang once
+ try {
+ outputFiles.addAll(yangProvider.addYangsToMetaInf(project, modelsInProject));
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed write model files for " + modelsInProject, e);
}
// add META_INF/services
- File generatedServicesDir = new GeneratedDirectories(project).getYangServicesDir();
- YangProvider.setResource(generatedServicesDir, project);
+ File generatedServicesDir = new File(new File(projectBuildDirectory, "generated-sources"), "spi");
+ ProjectFileAccess.addResourceDir(project, generatedServicesDir);
LOG.debug("{} Yang services files from: {} marked as resources: {}", LOG_PREFIX, generatedServicesDir,
META_INF_YANG_SERVICES_STRING_JAR);
}
}
- final var outputState = new YangToSourcesState(
- codeGenerators.stream()
- .collect(ImmutableMap.toImmutableMap(GeneratorTaskFactory::getIdentifier, GeneratorTaskFactory::arg)),
- projectYangs, dependencyYangs, new FileStateSet(ImmutableMap.copyOf(uniqueOutputFiles)));
+ // Reconcile push output files into project directory and acquire the execution state
+ final YangToSourcesState outputState;
+ try {
+ outputState = buildSupport.reconcileOutputFiles(buildContext, projectBuildDirectory, uniqueOutputFiles);
+ } catch (IOException e) {
+ throw new MojoFailureException("Failed to reconcile output files", e);
+ }
+ // Store execution state
try {
stateStorage.storeState(outputState);
} catch (IOException e) {
}
}
- private List<GeneratorTaskFactory> instantiateGenerators() throws MojoExecutionException {
+ private void wipeAllState(final YangToSourcesState prevState) throws MojoExecutionException {
+ try {
+ prevState.deleteOutputFiles();
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed to delete output files", e);
+ }
+ try {
+ stateStorage.deleteState();
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed to remove execution state", e);
+ }
+ }
+
+ private ImmutableList<@NonNull GeneratorTask> instantiateGenerators() throws MojoExecutionException {
// Search for available FileGenerator implementations
- final Map<String, FileGeneratorFactory> factories = Maps.uniqueIndex(
+ final var factories = Maps.uniqueIndex(
ServiceLoader.load(FileGeneratorFactory.class), FileGeneratorFactory::getIdentifier);
// FIXME: iterate over fileGeneratorArg instances (configuration), not factories (environment)
// Assign instantiate FileGenerators with appropriate configuration
- final var generators = new ArrayList<GeneratorTaskFactory>(factories.size());
- for (Entry<String, FileGeneratorFactory> entry : factories.entrySet()) {
- final String id = entry.getKey();
- FileGeneratorArg arg = fileGeneratorArgs.get(id);
+ final var builder = ImmutableList.<@NonNull GeneratorTask>builderWithExpectedSize(factories.size());
+ for (var entry : factories.entrySet()) {
+ final var id = entry.getKey();
+ var arg = fileGeneratorArgs.get(id);
if (arg == null) {
LOG.debug("{} No configuration for {}, using empty", LOG_PREFIX, id);
arg = new FileGeneratorArg(id);
}
+ final GeneratorTask task;
try {
- generators.add(GeneratorTaskFactory.of(entry.getValue(), arg));
+ task = new GeneratorTask(entry.getValue(), arg);
} catch (FileGeneratorException e) {
throw new MojoExecutionException("File generator " + id + " failed", e);
}
+ builder.add(task);
LOG.info("{} Code generator {} instantiated", LOG_PREFIX, id);
}
}
);
- return generators;
+ return builder.build();
}
@SuppressWarnings("checkstyle:illegalCatch")
- private Optional<ProcessorModuleReactor> createReactor(final List<File> yangFilesInProject,
+ private @NonNull ProcessorModuleReactor createReactor(final List<File> yangFilesInProject,
final YangParserConfiguration parserConfig, final Collection<ScannedDependency> dependencies,
- final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed) throws MojoExecutionException {
+ final List<Entry<YangTextSource, YangIRSchemaSource>> parsed) throws MojoExecutionException {
try {
- final List<YangTextSchemaSource> sourcesInProject = new ArrayList<>(yangFilesInProject.size());
- final YangParser parser = parserFactory.createParser(parserConfig);
- for (final Entry<YangTextSchemaSource, YangIRSchemaSource> entry : parsed) {
- final YangTextSchemaSource textSource = entry.getKey();
- final YangIRSchemaSource astSource = entry.getValue();
+ final var sourcesInProject = new ArrayList<YangTextSource>(yangFilesInProject.size());
+ final var parser = parserFactory.createParser(parserConfig);
+ for (var entry : parsed) {
+ final var textSource = entry.getKey();
+ final var astSource = entry.getValue();
parser.addSource(astSource);
- if (!astSource.getIdentifier().equals(textSource.getIdentifier())) {
+ if (!astSource.sourceId().equals(textSource.sourceId())) {
// AST indicates a different source identifier, make sure we use that
- sourcesInProject.add(YangTextSchemaSource.delegateForByteSource(astSource.getIdentifier(),
- textSource));
+ sourcesInProject.add(new DelegatedYangTextSource(astSource.sourceId(), textSource));
} else {
sourcesInProject.add(textSource);
}
}
- final ProcessorModuleReactor reactor = new ProcessorModuleReactor(parser, sourcesInProject, dependencies);
- LOG.debug("Initialized reactor {} with {}", reactor, yangFilesInProject);
- return Optional.of(reactor);
+ final var moduleReactor = new ProcessorModuleReactor(parser, sourcesInProject, dependencies);
+ LOG.debug("Initialized reactor {} with {}", moduleReactor, yangFilesInProject);
+ return moduleReactor;
} catch (IOException | YangSyntaxErrorException | RuntimeException e) {
// MojoExecutionException is thrown since execution cannot continue
LOG.error("{} Unable to parse YANG files from {}", LOG_PREFIX, yangFilesRootDir, e);
- Throwable rootCause = Throwables.getRootCause(e);
throw new MojoExecutionException(LOG_PREFIX + " Unable to parse YANG files from " + yangFilesRootDir,
- rootCause);
+ Throwables.getRootCause(e));
}
}