2 * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.yangtools.yang2sources.plugin;
10 import static com.google.common.base.Preconditions.checkState;
11 import static java.util.Objects.requireNonNull;
13 import com.google.common.annotations.VisibleForTesting;
14 import com.google.common.base.Stopwatch;
15 import com.google.common.base.Throwables;
16 import com.google.common.collect.ImmutableList;
17 import com.google.common.collect.ImmutableSet;
18 import com.google.common.collect.ImmutableSet.Builder;
19 import com.google.common.collect.Maps;
21 import java.io.IOException;
22 import java.nio.file.Files;
23 import java.nio.file.Path;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.Iterator;
27 import java.util.List;
29 import java.util.Map.Entry;
30 import java.util.Optional;
31 import java.util.ServiceLoader;
33 import java.util.stream.Collectors;
34 import org.apache.maven.plugin.MojoExecutionException;
35 import org.apache.maven.plugin.MojoFailureException;
36 import org.apache.maven.project.MavenProject;
37 import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorException;
38 import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorFactory;
39 import org.opendaylight.yangtools.yang.common.YangConstants;
40 import org.opendaylight.yangtools.yang.model.parser.api.YangParser;
41 import org.opendaylight.yangtools.yang.model.parser.api.YangParserException;
42 import org.opendaylight.yangtools.yang.model.parser.api.YangParserFactory;
43 import org.opendaylight.yangtools.yang.model.parser.api.YangSyntaxErrorException;
44 import org.opendaylight.yangtools.yang.model.repo.api.StatementParserMode;
45 import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
46 import org.opendaylight.yangtools.yang.parser.rfc7950.ir.IRSchemaSource;
47 import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToIRTransformer;
48 import org.opendaylight.yangtools.yang2sources.plugin.ConfigArg.CodeGeneratorArg;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
51 import org.sonatype.plexus.build.incremental.BuildContext;
52 import org.sonatype.plexus.build.incremental.DefaultBuildContext;
54 class YangToSourcesProcessor {
55 private static final Logger LOG = LoggerFactory.getLogger(YangToSourcesProcessor.class);
56 private static final YangParserFactory DEFAULT_PARSER_FACTORY;
59 final Iterator<YangParserFactory> it = ServiceLoader.load(YangParserFactory.class).iterator();
60 checkState(it.hasNext(), "Failed to find a YangParserFactory implementation");
61 DEFAULT_PARSER_FACTORY = it.next();
64 static final String LOG_PREFIX = "yang-to-sources:";
65 private static final String META_INF_STR = "META-INF";
66 private static final String YANG_STR = "yang";
68 static final String META_INF_YANG_STRING = META_INF_STR + File.separator + YANG_STR;
69 static final String META_INF_YANG_STRING_JAR = META_INF_STR + "/" + YANG_STR;
70 static final String META_INF_YANG_SERVICES_STRING_JAR = META_INF_STR + "/" + "services";
72 private final YangParserFactory parserFactory;
73 private final File yangFilesRootDir;
74 private final Set<File> excludedFiles;
75 private final List<CodeGeneratorArg> codeGeneratorArgs;
76 private final Map<String, FileGeneratorArg> fileGeneratorArgs;
77 private final MavenProject project;
78 private final boolean inspectDependencies;
79 private final BuildContext buildContext;
80 private final YangProvider yangProvider;
82 private YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
83 final Collection<File> excludedFiles, final List<CodeGeneratorArg> codeGeneratorArgs,
84 final List<FileGeneratorArg> fileGeneratorsArgs,
85 final MavenProject project, final boolean inspectDependencies, final YangProvider yangProvider) {
86 this.buildContext = requireNonNull(buildContext, "buildContext");
87 this.yangFilesRootDir = requireNonNull(yangFilesRootDir, "yangFilesRootDir");
88 this.excludedFiles = ImmutableSet.copyOf(excludedFiles);
89 this.codeGeneratorArgs = ImmutableList.copyOf(codeGeneratorArgs);
90 this.fileGeneratorArgs = Maps.uniqueIndex(fileGeneratorsArgs, FileGeneratorArg::getIdentifier);
91 this.project = requireNonNull(project);
92 this.inspectDependencies = inspectDependencies;
93 this.yangProvider = requireNonNull(yangProvider);
94 this.parserFactory = DEFAULT_PARSER_FACTORY;
98 YangToSourcesProcessor(final File yangFilesRootDir, final Collection<File> excludedFiles,
99 final List<CodeGeneratorArg> codeGenerators, final MavenProject project, final boolean inspectDependencies,
100 final YangProvider yangProvider) {
101 this(new DefaultBuildContext(), yangFilesRootDir, excludedFiles, codeGenerators, ImmutableList.of(),
102 project, inspectDependencies, yangProvider);
105 YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
106 final Collection<File> excludedFiles, final List<CodeGeneratorArg> codeGenerators,
107 final List<FileGeneratorArg> fileGenerators, final MavenProject project,
108 final boolean inspectDependencies) {
109 this(buildContext, yangFilesRootDir, excludedFiles, codeGenerators, fileGenerators, project,
110 inspectDependencies, YangProvider.getInstance());
113 public void execute() throws MojoExecutionException, MojoFailureException {
114 conditionalExecute(false);
117 void conditionalExecute(final boolean skip) throws MojoExecutionException, MojoFailureException {
119 * Collect all files which affect YANG context. This includes all
120 * files in current project and optionally any jars/files in the
123 final List<File> yangFilesInProject;
125 yangFilesInProject = listFiles(yangFilesRootDir, excludedFiles);
126 } catch (IOException e) {
127 throw new MojoFailureException("Failed to list project files", e);
130 if (yangFilesInProject.isEmpty()) {
131 // No files to process, skip.
132 LOG.info("{} No input files found", LOG_PREFIX);
136 // We need to instantiate all code generators to determine required import resolution mode
137 final List<GeneratorTaskFactory> codeGenerators = instantiateGenerators();
138 if (codeGenerators.isEmpty()) {
139 LOG.warn("{} No code generators provided", LOG_PREFIX);
143 final Set<StatementParserMode> parserModes = codeGenerators.stream()
144 .map(GeneratorTaskFactory::parserMode)
145 .collect(Collectors.toUnmodifiableSet());
147 // FIXME: store these files into state, so that we can verify/clean up
148 final Builder<File> files = ImmutableSet.builder();
149 for (StatementParserMode parserMode : parserModes) {
150 final Optional<ProcessorModuleReactor> optReactor = createReactor(yangFilesInProject, parserMode);
151 if (optReactor.isPresent()) {
152 final ProcessorModuleReactor reactor = optReactor.orElseThrow();
155 final Stopwatch watch = Stopwatch.createStarted();
156 final ContextHolder holder;
159 holder = reactor.toContext();
160 } catch (YangParserException e) {
161 throw new MojoFailureException("Failed to process reactor " + reactor, e);
162 } catch (IOException e) {
163 throw new MojoExecutionException("Failed to read reactor " + reactor, e);
166 LOG.info("{} {} YANG models processed in {}", LOG_PREFIX, holder.getContext().getModules().size(),
168 files.addAll(generateSources(holder, codeGenerators, parserMode));
170 LOG.info("{} Skipping YANG code generation because property yang.skip is true", LOG_PREFIX);
173 // FIXME: this is not right: we should be generating the models exactly once!
175 final Collection<YangTextSchemaSource> models = reactor.getModelsInProject();
177 yangProvider.addYangsToMetaInf(project, models);
178 } catch (IOException e) {
179 throw new MojoExecutionException("Failed write model files for " + models, e);
184 // add META_INF/services
185 File generatedServicesDir = new GeneratedDirectories(project).getYangServicesDir();
186 YangProvider.setResource(generatedServicesDir, project);
187 LOG.debug("{} Yang services files from: {} marked as resources: {}", LOG_PREFIX, generatedServicesDir,
188 META_INF_YANG_SERVICES_STRING_JAR);
191 private List<GeneratorTaskFactory> instantiateGenerators() throws MojoExecutionException, MojoFailureException {
192 final List<GeneratorTaskFactory> generators = new ArrayList<>(codeGeneratorArgs.size());
193 for (CodeGeneratorArg arg : codeGeneratorArgs) {
194 generators.add(CodeGeneratorTaskFactory.create(arg));
195 LOG.info("{} Code generator instantiated from {}", LOG_PREFIX, arg.getCodeGeneratorClass());
198 // Search for available FileGenerator implementations
199 final Map<String, FileGeneratorFactory> factories = Maps.uniqueIndex(
200 ServiceLoader.load(FileGeneratorFactory.class), FileGeneratorFactory::getIdentifier);
202 // Assign instantiate FileGenerators with appropriate configurate
203 for (Entry<String, FileGeneratorFactory> entry : factories.entrySet()) {
204 final String id = entry.getKey();
205 FileGeneratorArg arg = fileGeneratorArgs.get(id);
207 LOG.debug("{} No configuration for {}, using empty", LOG_PREFIX, id);
208 arg = new FileGeneratorArg(id);
212 generators.add(FileGeneratorTaskFactory.of(entry.getValue(), arg));
213 } catch (FileGeneratorException e) {
214 throw new MojoExecutionException("File generator " + id + " failed", e);
216 LOG.info("{} Code generator {} instantiated", LOG_PREFIX, id);
222 @SuppressWarnings("checkstyle:illegalCatch")
223 private Optional<ProcessorModuleReactor> createReactor(final List<File> yangFilesInProject,
224 final StatementParserMode parserMode) throws MojoExecutionException {
225 LOG.info("{} Inspecting {}", LOG_PREFIX, yangFilesRootDir);
227 final Collection<File> allFiles = new ArrayList<>(yangFilesInProject);
228 final Collection<ScannedDependency> dependencies;
229 if (inspectDependencies) {
230 dependencies = new ArrayList<>();
231 final Stopwatch watch = Stopwatch.createStarted();
234 ScannedDependency.scanDependencies(project).forEach(dep -> {
235 allFiles.add(dep.file());
236 dependencies.add(dep);
238 } catch (IOException e) {
239 LOG.error("{} Failed to scan dependencies", LOG_PREFIX, e);
240 throw new MojoExecutionException(LOG_PREFIX + " Failed to scan dependencies ", e);
242 LOG.info("{} Found {} dependencies in {}", LOG_PREFIX, dependencies.size(), watch);
244 dependencies = ImmutableList.of();
248 * Check if any of the listed files changed. If no changes occurred, simply return empty, which indicates
251 final Stopwatch watch = Stopwatch.createStarted();
252 if (!allFiles.stream().anyMatch(buildContext::hasDelta)) {
253 LOG.info("{} None of {} input files changed", LOG_PREFIX, allFiles.size());
254 return Optional.empty();
258 final List<Entry<YangTextSchemaSource, IRSchemaSource>> parsed = yangFilesInProject.parallelStream()
260 final YangTextSchemaSource textSource = YangTextSchemaSource.forFile(file);
262 return Map.entry(textSource,TextToIRTransformer.transformText(textSource));
263 } catch (YangSyntaxErrorException | IOException e) {
264 throw new IllegalArgumentException("Failed to parse " + file, e);
267 .collect(Collectors.toList());
269 // FIXME: all of the above checks need to be done just once
272 final List<YangTextSchemaSource> sourcesInProject = new ArrayList<>(yangFilesInProject.size());
273 final YangParser parser = parserFactory.createParser(parserMode);
274 for (final Entry<YangTextSchemaSource, IRSchemaSource> entry : parsed) {
275 final YangTextSchemaSource textSource = entry.getKey();
276 final IRSchemaSource astSource = entry.getValue();
277 parser.addSource(astSource);
279 if (!astSource.getIdentifier().equals(textSource.getIdentifier())) {
280 // AST indicates a different source identifier, make sure we use that
281 sourcesInProject.add(YangTextSchemaSource.delegateForByteSource(astSource.getIdentifier(),
284 sourcesInProject.add(textSource);
288 LOG.debug("Found project files: {}", yangFilesInProject);
289 LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
291 final ProcessorModuleReactor reactor = new ProcessorModuleReactor(parser, sourcesInProject, dependencies);
292 LOG.debug("Initialized reactor {} with {}", reactor, yangFilesInProject);
293 return Optional.of(reactor);
294 } catch (IOException | YangSyntaxErrorException | RuntimeException e) {
295 // MojoExecutionException is thrown since execution cannot continue
296 LOG.error("{} Unable to parse YANG files from {}", LOG_PREFIX, yangFilesRootDir, e);
297 Throwable rootCause = Throwables.getRootCause(e);
298 throw new MojoExecutionException(LOG_PREFIX + " Unable to parse YANG files from " + yangFilesRootDir,
303 private static List<File> listFiles(final File root, final Collection<File> excludedFiles)
305 if (!root.isDirectory()) {
306 LOG.warn("{} YANG source directory {} not found. No code will be generated.", LOG_PREFIX, root);
307 return ImmutableList.of();
310 return Files.walk(root.toPath()).map(Path::toFile).filter(File::isFile).filter(f -> {
311 if (excludedFiles.contains(f)) {
312 LOG.info("{} YANG file excluded {}", LOG_PREFIX, f);
316 }).filter(f -> f.getName().endsWith(YangConstants.RFC6020_YANG_FILE_EXTENSION)).collect(Collectors.toList());
320 * Call generate on every generator from plugin configuration.
322 private Set<File> generateSources(final ContextHolder context, final Collection<GeneratorTaskFactory> generators,
323 final StatementParserMode parserMode) throws MojoFailureException {
324 final Builder<File> allFiles = ImmutableSet.builder();
325 for (GeneratorTaskFactory factory : generators) {
326 if (!parserMode.equals(factory.parserMode())) {
330 final Stopwatch sw = Stopwatch.createStarted();
331 final GeneratorTask<?> task = factory.createTask(project, context);
332 LOG.debug("{} Task {} initialized in {}", LOG_PREFIX, task, sw);
334 final Collection<File> files;
336 files = task.execute(buildContext);
337 } catch (FileGeneratorException | IOException e) {
338 throw new MojoFailureException(LOG_PREFIX + " Generator " + factory + " failed", e);
341 LOG.debug("{} Sources generated by {}: {}", LOG_PREFIX, factory.generatorName(), files);
342 LOG.info("{} Sources generated by {}: {} in {}", LOG_PREFIX, factory.generatorName(),
343 files == null ? 0 : files.size(), sw);
344 allFiles.addAll(files);
347 return allFiles.build();