2 * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.yangtools.yang2sources.plugin;
10 import static java.util.Objects.requireNonNull;
12 import com.google.common.annotations.VisibleForTesting;
13 import com.google.common.base.Stopwatch;
14 import com.google.common.base.Throwables;
15 import com.google.common.collect.ImmutableList;
16 import com.google.common.collect.ImmutableMap;
17 import com.google.common.collect.ImmutableSet;
18 import com.google.common.collect.Maps;
20 import java.io.IOException;
21 import java.nio.file.Files;
22 import java.nio.file.Path;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.LinkedHashMap;
26 import java.util.List;
28 import java.util.Map.Entry;
29 import java.util.NoSuchElementException;
30 import java.util.ServiceLoader;
32 import java.util.function.Function;
33 import java.util.stream.Collectors;
34 import org.apache.maven.plugin.MojoExecutionException;
35 import org.apache.maven.plugin.MojoFailureException;
36 import org.apache.maven.project.MavenProject;
37 import org.eclipse.jdt.annotation.NonNull;
38 import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorException;
39 import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorFactory;
40 import org.opendaylight.yangtools.yang.common.YangConstants;
41 import org.opendaylight.yangtools.yang.model.repo.api.YangIRSchemaSource;
42 import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
43 import org.opendaylight.yangtools.yang.parser.api.YangParserConfiguration;
44 import org.opendaylight.yangtools.yang.parser.api.YangParserException;
45 import org.opendaylight.yangtools.yang.parser.api.YangParserFactory;
46 import org.opendaylight.yangtools.yang.parser.api.YangSyntaxErrorException;
47 import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToIRTransformer;
48 import org.slf4j.Logger;
49 import org.slf4j.LoggerFactory;
50 import org.sonatype.plexus.build.incremental.BuildContext;
51 import org.sonatype.plexus.build.incremental.DefaultBuildContext;
53 // FIXME: rename to Execution
55 class YangToSourcesProcessor {
56 private static final Logger LOG = LoggerFactory.getLogger(YangToSourcesProcessor.class);
57 private static final YangParserFactory DEFAULT_PARSER_FACTORY;
61 DEFAULT_PARSER_FACTORY = ServiceLoader.load(YangParserFactory.class).iterator().next();
62 } catch (NoSuchElementException e) {
63 throw new IllegalStateException("Failed to find a YangParserFactory implementation", e);
67 private static final String META_INF_STR = "META-INF";
68 private static final String YANG_STR = "yang";
70 static final String LOG_PREFIX = "yang-to-sources:";
71 static final String META_INF_YANG_STRING = META_INF_STR + File.separator + YANG_STR;
72 static final String META_INF_YANG_STRING_JAR = META_INF_STR + "/" + YANG_STR;
73 static final String META_INF_YANG_SERVICES_STRING_JAR = META_INF_STR + "/" + "services";
75 private static final YangProvider YANG_PROVIDER = (project, modelsInProject) -> {
76 final var generatedYangDir =
77 // FIXME: why are we generating these in "generated-sources"? At the end of the day YANG files are more
78 // resources (except we do not them to be subject to filtering)
79 new File(new File(project.getBuild().getDirectory(), "generated-sources"), "yang");
80 LOG.debug("Generated dir {}", generatedYangDir);
82 // copy project's src/main/yang/*.yang to ${project.builddir}/generated-sources/yang/META-INF/yang/
83 // This honors setups like a Eclipse-profile derived one
84 final var withMetaInf = new File(generatedYangDir, YangToSourcesProcessor.META_INF_YANG_STRING);
85 Files.createDirectories(withMetaInf.toPath());
87 final var stateListBuilder = ImmutableList.<FileState>builderWithExpectedSize(modelsInProject.size());
88 for (var source : modelsInProject) {
89 final File file = new File(withMetaInf, source.getIdentifier().toYangFilename());
90 stateListBuilder.add(FileState.ofWrittenFile(file, source::copyTo));
91 LOG.debug("Created file {} for {}", file, source.getIdentifier());
94 ProjectFileAccess.addResourceDir(project, generatedYangDir);
95 LOG.debug("{} YANG files marked as resources: {}", YangToSourcesProcessor.LOG_PREFIX, generatedYangDir);
97 return stateListBuilder.build();
100 private final YangParserFactory parserFactory;
101 private final File yangFilesRootDir;
102 private final Set<File> excludedFiles;
103 private final ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs;
104 private final @NonNull MavenProject project;
105 private final boolean inspectDependencies;
106 private final @NonNull BuildContext buildContext;
107 private final YangProvider yangProvider;
108 private final StateStorage stateStorage;
109 private final String projectBuildDirectory;
111 private YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
112 final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGeneratorsArgs,
113 final MavenProject project, final boolean inspectDependencies, final YangProvider yangProvider) {
114 this.buildContext = requireNonNull(buildContext, "buildContext");
115 this.yangFilesRootDir = requireNonNull(yangFilesRootDir, "yangFilesRootDir");
116 this.excludedFiles = ImmutableSet.copyOf(excludedFiles);
117 //FIXME multiple FileGeneratorArg entries of same identifier became one here
118 fileGeneratorArgs = Maps.uniqueIndex(fileGeneratorsArgs, FileGeneratorArg::getIdentifier);
119 this.project = requireNonNull(project);
120 this.inspectDependencies = inspectDependencies;
121 this.yangProvider = requireNonNull(yangProvider);
122 projectBuildDirectory = project.getBuild().getDirectory();
123 stateStorage = StateStorage.of(buildContext, stateFilePath(projectBuildDirectory));
124 parserFactory = DEFAULT_PARSER_FACTORY;
128 YangToSourcesProcessor(final File yangFilesRootDir, final List<FileGeneratorArg> fileGenerators,
129 final MavenProject project, final YangProvider yangProvider) {
130 this(new DefaultBuildContext(), yangFilesRootDir, List.of(), List.of(), project, false, yangProvider);
133 YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
134 final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGenerators,
135 final MavenProject project, final boolean inspectDependencies) {
136 this(buildContext, yangFilesRootDir, excludedFiles, fileGenerators, project, inspectDependencies,
140 void execute() throws MojoExecutionException, MojoFailureException {
141 YangToSourcesState prevState;
143 prevState = stateStorage.loadState();
144 } catch (IOException e) {
145 throw new MojoFailureException("Failed to restore execution state", e);
147 if (prevState == null) {
148 LOG.debug("{} no previous execution state present", LOG_PREFIX);
149 prevState = new YangToSourcesState(ImmutableMap.of(),
150 FileStateSet.empty(), FileStateSet.empty(), FileStateSet.empty());
153 // Collect all files in the current project.
154 final List<File> yangFilesInProject;
156 yangFilesInProject = listFiles(yangFilesRootDir, excludedFiles);
157 } catch (IOException e) {
158 throw new MojoFailureException("Failed to list project files", e);
161 if (yangFilesInProject.isEmpty()) {
162 // No files to process, skip.
163 LOG.info("{} No input files found", LOG_PREFIX);
164 wipeAllState(prevState);
168 // We need to instantiate all code generators to determine required import resolution mode
169 final var codeGenerators = instantiateGenerators();
170 if (codeGenerators.isEmpty()) {
171 LOG.warn("{} No code generators provided", LOG_PREFIX);
172 wipeAllState(prevState);
176 LOG.info("{} Inspecting {}", LOG_PREFIX, yangFilesRootDir);
178 // All files which affect YANG context. This minimally includes all files in the current project, but optionally
179 // may include any YANG files in the dependencies.
180 final List<ScannedDependency> dependencies;
181 if (inspectDependencies) {
182 final Stopwatch watch = Stopwatch.createStarted();
184 dependencies = ScannedDependency.scanDependencies(project);
185 } catch (IOException e) {
186 LOG.error("{} Failed to scan dependencies", LOG_PREFIX, e);
187 throw new MojoExecutionException(LOG_PREFIX + " Failed to scan dependencies ", e);
189 LOG.info("{} Found {} dependencies in {}", LOG_PREFIX, dependencies.size(), watch);
191 dependencies = List.of();
194 // Determine hash/size of YANG input files and dependencies in parallel
195 final var hashTimer = Stopwatch.createStarted();
196 final var projectYangs = new FileStateSet(yangFilesInProject.parallelStream()
199 return FileState.ofFile(file);
200 } catch (IOException e) {
201 throw new IllegalStateException("Failed to read " + file, e);
204 .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
205 // TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
206 // not the entire file
207 final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
208 .map(ScannedDependency::file)
211 return FileState.ofFile(file);
212 } catch (IOException e) {
213 throw new IllegalStateException("Failed to read " + file, e);
216 .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
217 LOG.debug("{} Input state determined in {}", LOG_PREFIX, hashTimer);
219 // We have collected our current inputs and previous state. Instantiate a support object which will guide us for
220 // the rest of the way.
221 final var buildSupport = new IncrementalBuildSupport(prevState,
222 codeGenerators.stream()
223 .collect(ImmutableMap.toImmutableMap(GeneratorTask::getIdentifier, GeneratorTask::arg)),
224 projectYangs, dependencyYangs);
226 // Check if any inputs changed, which is supposed to be fast. If they did not, we need to also validate our
227 // our previous are also up-to-date.
228 if (!buildSupport.inputsChanged()) {
229 final boolean outputsChanged;
231 outputsChanged = buildSupport.outputsChanged(projectBuildDirectory);
232 } catch (IOException e) {
233 throw new MojoFailureException("Failed to reconcile generation outputs", e);
236 if (!outputsChanged) {
237 // FIXME: YANGTOOLS-745: still need to add all resources/directories to maven project
238 LOG.info("{}: Everything is up to date, nothing to do", LOG_PREFIX);
243 final Stopwatch watch = Stopwatch.createStarted();
245 final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
247 final YangTextSchemaSource textSource = YangTextSchemaSource.forPath(file.toPath());
249 return Map.entry(textSource, TextToIRTransformer.transformText(textSource));
250 } catch (YangSyntaxErrorException | IOException e) {
251 throw new IllegalArgumentException("Failed to parse " + file, e);
254 .collect(Collectors.toList());
255 LOG.debug("Found project files: {}", yangFilesInProject);
256 LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
258 final var outputFiles = ImmutableList.<FileState>builder();
259 Collection<YangTextSchemaSource> modelsInProject = null;
260 for (var parserConfig : codeGenerators.stream().map(GeneratorTask::parserConfig).collect(Collectors.toSet())) {
261 final var moduleReactor = createReactor(yangFilesInProject, parserConfig, dependencies, parsed);
262 final var yangSw = Stopwatch.createStarted();
264 final ContextHolder holder;
266 holder = moduleReactor.toContext();
267 } catch (YangParserException e) {
268 throw new MojoFailureException("Failed to process reactor " + moduleReactor, e);
269 } catch (IOException e) {
270 throw new MojoExecutionException("Failed to read reactor " + moduleReactor, e);
272 LOG.info("{} {} YANG models processed in {}", LOG_PREFIX, holder.getContext().getModules().size(), yangSw);
274 for (var factory : codeGenerators) {
275 if (!parserConfig.equals(factory.parserConfig())) {
279 final var genSw = Stopwatch.createStarted();
280 final List<FileState> files;
282 files = factory.execute(project, buildContext, holder);
283 } catch (FileGeneratorException e) {
284 throw new MojoFailureException(LOG_PREFIX + " Generator " + factory + " failed", e);
287 outputFiles.addAll(files);
288 LOG.info("{} Sources generated by {}: {} in {}", LOG_PREFIX, factory.generatorName(), files.size(),
292 if (modelsInProject == null) {
293 // FIXME: this is an invariant, we should prepare these separately
294 modelsInProject = moduleReactor.getModelsInProject();
298 // add META_INF/yang once
300 outputFiles.addAll(yangProvider.addYangsToMetaInf(project, modelsInProject));
301 } catch (IOException e) {
302 throw new MojoExecutionException("Failed write model files for " + modelsInProject, e);
305 // add META_INF/services
306 File generatedServicesDir = new File(new File(projectBuildDirectory, "generated-sources"), "spi");
307 ProjectFileAccess.addResourceDir(project, generatedServicesDir);
308 LOG.debug("{} Yang services files from: {} marked as resources: {}", LOG_PREFIX, generatedServicesDir,
309 META_INF_YANG_SERVICES_STRING_JAR);
311 final var uniqueOutputFiles = new LinkedHashMap<String, FileState>();
312 for (var fileHash : outputFiles.build()) {
313 final var prev = uniqueOutputFiles.putIfAbsent(fileHash.path(), fileHash);
315 throw new MojoFailureException("Duplicate files " + prev + " and " + fileHash);
319 // Reconcile push output files into project directory and acquire the execution state
320 final YangToSourcesState outputState;
322 outputState = buildSupport.reconcileOutputFiles(buildContext, projectBuildDirectory, uniqueOutputFiles);
323 } catch (IOException e) {
324 throw new MojoFailureException("Failed to reconcile output files", e);
327 // Store execution state
329 stateStorage.storeState(outputState);
330 } catch (IOException e) {
331 throw new MojoFailureException("Failed to store execution state", e);
335 private void wipeAllState(final YangToSourcesState prevState) throws MojoExecutionException {
337 prevState.deleteOutputFiles();
338 } catch (IOException e) {
339 throw new MojoExecutionException("Failed to delete output files", e);
342 stateStorage.deleteState();
343 } catch (IOException e) {
344 throw new MojoExecutionException("Failed to remove execution state", e);
348 private List<GeneratorTask> instantiateGenerators() throws MojoExecutionException {
349 // Search for available FileGenerator implementations
350 final Map<String, FileGeneratorFactory> factories = Maps.uniqueIndex(
351 ServiceLoader.load(FileGeneratorFactory.class), FileGeneratorFactory::getIdentifier);
353 // FIXME: iterate over fileGeneratorArg instances (configuration), not factories (environment)
354 // Assign instantiate FileGenerators with appropriate configuration
355 final var generators = new ArrayList<GeneratorTask>(factories.size());
356 for (Entry<String, FileGeneratorFactory> entry : factories.entrySet()) {
357 final String id = entry.getKey();
358 FileGeneratorArg arg = fileGeneratorArgs.get(id);
360 LOG.debug("{} No configuration for {}, using empty", LOG_PREFIX, id);
361 arg = new FileGeneratorArg(id);
364 final GeneratorTask task;
366 task = new GeneratorTask(entry.getValue(), arg);
367 } catch (FileGeneratorException e) {
368 throw new MojoExecutionException("File generator " + id + " failed", e);
370 generators.add(task);
371 LOG.info("{} Code generator {} instantiated", LOG_PREFIX, id);
374 // Notify if no factory found for defined identifiers
375 fileGeneratorArgs.keySet().forEach(
376 fileGenIdentifier -> {
377 if (!factories.containsKey(fileGenIdentifier)) {
378 LOG.warn("{} No generator found for identifier {}", LOG_PREFIX, fileGenIdentifier);
386 @SuppressWarnings("checkstyle:illegalCatch")
387 private @NonNull ProcessorModuleReactor createReactor(final List<File> yangFilesInProject,
388 final YangParserConfiguration parserConfig, final Collection<ScannedDependency> dependencies,
389 final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed) throws MojoExecutionException {
392 final var sourcesInProject = new ArrayList<YangTextSchemaSource>(yangFilesInProject.size());
393 final var parser = parserFactory.createParser(parserConfig);
394 for (var entry : parsed) {
395 final var textSource = entry.getKey();
396 final var astSource = entry.getValue();
397 parser.addSource(astSource);
399 if (!astSource.getIdentifier().equals(textSource.getIdentifier())) {
400 // AST indicates a different source identifier, make sure we use that
401 sourcesInProject.add(YangTextSchemaSource.delegateForByteSource(astSource.getIdentifier(),
404 sourcesInProject.add(textSource);
408 final var moduleReactor = new ProcessorModuleReactor(parser, sourcesInProject, dependencies);
409 LOG.debug("Initialized reactor {} with {}", moduleReactor, yangFilesInProject);
410 return moduleReactor;
411 } catch (IOException | YangSyntaxErrorException | RuntimeException e) {
412 // MojoExecutionException is thrown since execution cannot continue
413 LOG.error("{} Unable to parse YANG files from {}", LOG_PREFIX, yangFilesRootDir, e);
414 throw new MojoExecutionException(LOG_PREFIX + " Unable to parse YANG files from " + yangFilesRootDir,
415 Throwables.getRootCause(e));
419 private static ImmutableList<File> listFiles(final File root, final Collection<File> excludedFiles)
421 if (!root.isDirectory()) {
422 LOG.warn("{} YANG source directory {} not found. No code will be generated.", LOG_PREFIX, root);
423 return ImmutableList.of();
426 return Files.walk(root.toPath())
428 .filter(File::isFile)
430 if (excludedFiles.contains(f)) {
431 LOG.info("{} YANG file excluded {}", LOG_PREFIX, f);
436 .filter(f -> f.getName().endsWith(YangConstants.RFC6020_YANG_FILE_EXTENSION))
437 .collect(ImmutableList.toImmutableList());
441 static @NonNull Path stateFilePath(final String projectBuildDirectory) {
442 // ${project.build.directory}/maven-status/yang-maven-plugin/execution.state
443 return Path.of(projectBuildDirectory, "maven-status", "yang-maven-plugin", "execution.state");