plugin/yang-maven-plugin/src/main/java/org/opendaylight/yangtools/yang2sources/plugin/YangToSourcesProcessor.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.yangtools.yang2sources.plugin;
   9
  10 import static com.google.common.base.Preconditions.checkState;
  11 import static java.util.Objects.requireNonNull;
  12
  13 import com.google.common.annotations.VisibleForTesting;
  14 import com.google.common.base.Stopwatch;
  15 import com.google.common.base.Throwables;
  16 import com.google.common.collect.ImmutableList;
  17 import com.google.common.collect.ImmutableMap;
  18 import com.google.common.collect.ImmutableSet;
  19 import com.google.common.collect.Maps;
  20 import java.io.File;
  21 import java.io.IOException;
  22 import java.nio.file.Files;
  23 import java.nio.file.Path;
  24 import java.util.ArrayList;
  25 import java.util.Collection;
  26 import java.util.Iterator;
  27 import java.util.LinkedHashMap;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Map.Entry;
  31 import java.util.Optional;
  32 import java.util.ServiceLoader;
  33 import java.util.Set;
  34 import java.util.function.Function;
  35 import java.util.stream.Collectors;
  36 import java.util.stream.Stream;
  37 import org.apache.maven.plugin.MojoExecutionException;
  38 import org.apache.maven.plugin.MojoFailureException;
  39 import org.apache.maven.project.MavenProject;
  40 import org.eclipse.jdt.annotation.NonNull;
  41 import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorException;
  42 import org.opendaylight.yangtools.plugin.generator.api.FileGeneratorFactory;
  43 import org.opendaylight.yangtools.yang.common.YangConstants;
  44 import org.opendaylight.yangtools.yang.model.repo.api.YangIRSchemaSource;
  45 import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
  46 import org.opendaylight.yangtools.yang.parser.api.YangParser;
  47 import org.opendaylight.yangtools.yang.parser.api.YangParserConfiguration;
  48 import org.opendaylight.yangtools.yang.parser.api.YangParserException;
  49 import org.opendaylight.yangtools.yang.parser.api.YangParserFactory;
  50 import org.opendaylight.yangtools.yang.parser.api.YangSyntaxErrorException;
  51 import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToIRTransformer;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54 import org.sonatype.plexus.build.incremental.BuildContext;
  55 import org.sonatype.plexus.build.incremental.DefaultBuildContext;
  56
  57 class YangToSourcesProcessor {
  58     private static final Logger LOG = LoggerFactory.getLogger(YangToSourcesProcessor.class);
  59     private static final YangParserFactory DEFAULT_PARSER_FACTORY;
  60
  61     static {
  62         final Iterator<YangParserFactory> it = ServiceLoader.load(YangParserFactory.class).iterator();
  63         checkState(it.hasNext(), "Failed to find a YangParserFactory implementation");
  64         DEFAULT_PARSER_FACTORY = it.next();
  65     }
  66
  67     static final String LOG_PREFIX = "yang-to-sources:";
  68     private static final String META_INF_STR = "META-INF";
  69     private static final String YANG_STR = "yang";
  70
  71     static final String META_INF_YANG_STRING = META_INF_STR + File.separator + YANG_STR;
  72     static final String META_INF_YANG_STRING_JAR = META_INF_STR + "/" + YANG_STR;
  73     static final String META_INF_YANG_SERVICES_STRING_JAR = META_INF_STR + "/" + "services";
  74
  75     private final YangParserFactory parserFactory;
  76     private final File yangFilesRootDir;
  77     private final Set<File> excludedFiles;
  78     private final ImmutableMap<String, FileGeneratorArg> fileGeneratorArgs;
  79     private final @NonNull MavenProject project;
  80     private final boolean inspectDependencies;
  81     private final BuildContext buildContext;
  82     private final YangProvider yangProvider;
  83     private final StateStorage stateStorage;
  84
  85     private YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
  86             final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGeneratorsArgs,
  87             final MavenProject project, final boolean inspectDependencies, final YangProvider yangProvider) {
  88         this.buildContext = requireNonNull(buildContext, "buildContext");
  89         this.yangFilesRootDir = requireNonNull(yangFilesRootDir, "yangFilesRootDir");
  90         this.excludedFiles = ImmutableSet.copyOf(excludedFiles);
  91         //FIXME multiple FileGeneratorArg entries of same identifier became one here
  92         fileGeneratorArgs = Maps.uniqueIndex(fileGeneratorsArgs, FileGeneratorArg::getIdentifier);
  93         this.project = requireNonNull(project);
  94         this.inspectDependencies = inspectDependencies;
  95         this.yangProvider = requireNonNull(yangProvider);
  96         stateStorage = StateStorage.of(buildContext, stateFilePath(project.getBuild().getDirectory()));
  97         parserFactory = DEFAULT_PARSER_FACTORY;
  98     }
  99
 100     @VisibleForTesting
 101     YangToSourcesProcessor(final File yangFilesRootDir, final Collection<File> excludedFiles,
 102             final List<FileGeneratorArg> fileGenerators, final MavenProject project, final boolean inspectDependencies,
 103             final YangProvider yangProvider) {
 104         this(new DefaultBuildContext(), yangFilesRootDir, excludedFiles, ImmutableList.of(),
 105             project, inspectDependencies, yangProvider);
 106     }
 107
 108     YangToSourcesProcessor(final BuildContext buildContext, final File yangFilesRootDir,
 109             final Collection<File> excludedFiles, final List<FileGeneratorArg> fileGenerators,
 110             final MavenProject project, final boolean inspectDependencies) {
 111         this(buildContext, yangFilesRootDir, excludedFiles, fileGenerators, project, inspectDependencies,
 112             YangProvider.getInstance());
 113     }
 114
 115     void execute() throws MojoExecutionException, MojoFailureException {
 116         final YangToSourcesState prevState;
 117         try {
 118             prevState = stateStorage.loadState();
 119         } catch (IOException e) {
 120             throw new MojoFailureException("Failed to restore execution state", e);
 121         }
 122         if (prevState == null) {
 123             LOG.debug("{} no previous execution state present", LOG_PREFIX);
 124         }
 125
 126         // Collect all files in the current project.
 127         final List<File> yangFilesInProject;
 128         try {
 129             yangFilesInProject = listFiles(yangFilesRootDir, excludedFiles);
 130         } catch (IOException e) {
 131             throw new MojoFailureException("Failed to list project files", e);
 132         }
 133
 134         if (yangFilesInProject.isEmpty()) {
 135             // No files to process, skip.
 136             LOG.info("{} No input files found", LOG_PREFIX);
 137             return;
 138         }
 139
 140         // We need to instantiate all code generators to determine required import resolution mode
 141         final List<GeneratorTaskFactory> codeGenerators = instantiateGenerators();
 142         if (codeGenerators.isEmpty()) {
 143             LOG.warn("{} No code generators provided", LOG_PREFIX);
 144             return;
 145         }
 146
 147         final Set<YangParserConfiguration> parserConfigs = codeGenerators.stream()
 148             .map(GeneratorTaskFactory::parserConfig)
 149             .collect(Collectors.toUnmodifiableSet());
 150
 151         LOG.info("{} Inspecting {}", LOG_PREFIX, yangFilesRootDir);
 152
 153         // All files which affect YANG context. This minimally includes all files in the current project, but optionally
 154         // may include any YANG files in the dependencies.
 155         final List<ScannedDependency> dependencies;
 156         if (inspectDependencies) {
 157             final Stopwatch watch = Stopwatch.createStarted();
 158             try {
 159                 dependencies = ScannedDependency.scanDependencies(project);
 160             } catch (IOException e) {
 161                 LOG.error("{} Failed to scan dependencies", LOG_PREFIX, e);
 162                 throw new MojoExecutionException(LOG_PREFIX + " Failed to scan dependencies ", e);
 163             }
 164             LOG.info("{} Found {} dependencies in {}", LOG_PREFIX, dependencies.size(), watch);
 165         } else {
 166             dependencies = ImmutableList.of();
 167         }
 168
 169         /*
 170          * Check if any of the listed files changed. If no changes occurred, simply return empty, which indicates
 171          * end of execution.
 172          */
 173         // FIXME: YANGTOOLS-745: remove this check FileStates instead
 174         if (!Stream.concat(yangFilesInProject.stream(), dependencies.stream().map(ScannedDependency::file))
 175                 .anyMatch(buildContext::hasDelta)) {
 176             LOG.info("{} None of {} input files changed", LOG_PREFIX, yangFilesInProject.size() + dependencies.size());
 177             return;
 178         }
 179
 180         final Stopwatch watch = Stopwatch.createStarted();
 181         // Determine hash/size of YANG input files in parallel
 182         final var projectYangs = new FileStateSet(yangFilesInProject.parallelStream()
 183             .map(file -> {
 184                 try {
 185                     return FileState.ofFile(file);
 186                 } catch (IOException e) {
 187                     throw new IllegalStateException("Failed to read " + file, e);
 188                 }
 189             })
 190             .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
 191
 192         final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed = yangFilesInProject.parallelStream()
 193             .map(file -> {
 194                 final YangTextSchemaSource textSource = YangTextSchemaSource.forPath(file.toPath());
 195                 try {
 196                     return Map.entry(textSource, TextToIRTransformer.transformText(textSource));
 197                 } catch (YangSyntaxErrorException | IOException e) {
 198                     throw new IllegalArgumentException("Failed to parse " + file, e);
 199                 }
 200             })
 201             .collect(Collectors.toList());
 202         LOG.debug("Found project files: {}", yangFilesInProject);
 203         LOG.info("{} Project model files found: {} in {}", LOG_PREFIX, yangFilesInProject.size(), watch);
 204
 205         // Determine hash/size of dependency files
 206         // TODO: this produces false positives for Jar files -- there we want to capture the contents of the YANG files,
 207         //       not the entire file
 208         final var dependencyYangs = new FileStateSet(dependencies.parallelStream()
 209             .map(ScannedDependency::file)
 210             .map(file -> {
 211                 try {
 212                     return FileState.ofFile(file);
 213                 } catch (IOException e) {
 214                     throw new IllegalStateException("Failed to read " + file, e);
 215                 }
 216             })
 217             .collect(ImmutableMap.toImmutableMap(FileState::path, Function.identity())));
 218
 219         final var outputFiles = ImmutableList.<FileState>builder();
 220         boolean sourcesPersisted = false;
 221
 222         for (YangParserConfiguration parserConfig : parserConfigs) {
 223             final Optional<ProcessorModuleReactor> optReactor = createReactor(yangFilesInProject,
 224                 parserConfig, dependencies, parsed);
 225             if (optReactor.isPresent()) {
 226                 final ProcessorModuleReactor reactor = optReactor.orElseThrow();
 227
 228                 final Stopwatch sw = Stopwatch.createStarted();
 229                 final ContextHolder holder;
 230
 231                 try {
 232                     holder = reactor.toContext();
 233                 } catch (YangParserException e) {
 234                     throw new MojoFailureException("Failed to process reactor " + reactor, e);
 235                 } catch (IOException e) {
 236                     throw new MojoExecutionException("Failed to read reactor " + reactor, e);
 237                 }
 238
 239                 LOG.info("{} {} YANG models processed in {}", LOG_PREFIX, holder.getContext().getModules().size(), sw);
 240
 241                 for (GeneratorTaskFactory factory : codeGenerators) {
 242                     if (!parserConfig.equals(factory.parserConfig())) {
 243                         continue;
 244                     }
 245
 246                     final Stopwatch genSw = Stopwatch.createStarted();
 247                     final GeneratorTask task = factory.createTask(project, holder);
 248                     LOG.debug("{} Task {} initialized in {}", LOG_PREFIX, task, genSw);
 249
 250                     final List<FileState> files;
 251                     try {
 252                         files = task.execute(buildContext);
 253                     } catch (FileGeneratorException | IOException e) {
 254                         throw new MojoFailureException(LOG_PREFIX + " Generator " + factory + " failed", e);
 255                     }
 256
 257                     outputFiles.addAll(files);
 258
 259                     LOG.info("{} Sources generated by {}: {} in {}", LOG_PREFIX, factory.generatorName(), files.size(),
 260                         genSw);
 261                 }
 262
 263                 if (!sourcesPersisted) {
 264                     // add META_INF/yang once
 265                     final var models = reactor.getModelsInProject();
 266                     final Collection<FileState> sourceFileStates;
 267                     try {
 268                         sourceFileStates = yangProvider.addYangsToMetaInf(project, models);
 269                     } catch (IOException e) {
 270                         throw new MojoExecutionException("Failed write model files for " + models, e);
 271                     }
 272
 273                     sourcesPersisted = true;
 274                     outputFiles.addAll(sourceFileStates);
 275                 }
 276             }
 277         }
 278
 279         // add META_INF/services
 280         File generatedServicesDir = new GeneratedDirectories(project).getYangServicesDir();
 281         YangProvider.setResource(generatedServicesDir, project);
 282         LOG.debug("{} Yang services files from: {} marked as resources: {}", LOG_PREFIX, generatedServicesDir,
 283             META_INF_YANG_SERVICES_STRING_JAR);
 284
 285         final var uniqueOutputFiles = new LinkedHashMap<String, FileState>();
 286         for (var fileHash : outputFiles.build()) {
 287             final var prev = uniqueOutputFiles.putIfAbsent(fileHash.path(), fileHash);
 288             if (prev != null) {
 289                 throw new MojoFailureException("Duplicate files " + prev + " and " + fileHash);
 290             }
 291         }
 292
 293         final var outputState = new YangToSourcesState(
 294             codeGenerators.stream()
 295                 .collect(ImmutableMap.toImmutableMap(GeneratorTaskFactory::getIdentifier, GeneratorTaskFactory::arg)),
 296             projectYangs, dependencyYangs, new FileStateSet(ImmutableMap.copyOf(uniqueOutputFiles)));
 297
 298         try {
 299             stateStorage.storeState(outputState);
 300         } catch (IOException e) {
 301             throw new MojoFailureException("Failed to store execution state", e);
 302         }
 303     }
 304
 305     private List<GeneratorTaskFactory> instantiateGenerators() throws MojoExecutionException {
 306         // Search for available FileGenerator implementations
 307         final Map<String, FileGeneratorFactory> factories = Maps.uniqueIndex(
 308             ServiceLoader.load(FileGeneratorFactory.class), FileGeneratorFactory::getIdentifier);
 309
 310         // FIXME: iterate over fileGeneratorArg instances (configuration), not factories (environment)
 311         // Assign instantiate FileGenerators with appropriate configuration
 312         final var generators = new ArrayList<GeneratorTaskFactory>(factories.size());
 313         for (Entry<String, FileGeneratorFactory> entry : factories.entrySet()) {
 314             final String id = entry.getKey();
 315             FileGeneratorArg arg = fileGeneratorArgs.get(id);
 316             if (arg == null) {
 317                 LOG.debug("{} No configuration for {}, using empty", LOG_PREFIX, id);
 318                 arg = new FileGeneratorArg(id);
 319             }
 320
 321             try {
 322                 generators.add(GeneratorTaskFactory.of(entry.getValue(), arg));
 323             } catch (FileGeneratorException e) {
 324                 throw new MojoExecutionException("File generator " + id + " failed", e);
 325             }
 326             LOG.info("{} Code generator {} instantiated", LOG_PREFIX, id);
 327         }
 328
 329         // Notify if no factory found for defined identifiers
 330         fileGeneratorArgs.keySet().forEach(
 331             fileGenIdentifier -> {
 332                 if (!factories.containsKey(fileGenIdentifier)) {
 333                     LOG.warn("{} No generator found for identifier {}", LOG_PREFIX, fileGenIdentifier);
 334                 }
 335             }
 336         );
 337
 338         return generators;
 339     }
 340
 341     @SuppressWarnings("checkstyle:illegalCatch")
 342     private Optional<ProcessorModuleReactor> createReactor(final List<File> yangFilesInProject,
 343             final YangParserConfiguration parserConfig, final Collection<ScannedDependency> dependencies,
 344             final List<Entry<YangTextSchemaSource, YangIRSchemaSource>> parsed) throws MojoExecutionException {
 345
 346         try {
 347             final List<YangTextSchemaSource> sourcesInProject = new ArrayList<>(yangFilesInProject.size());
 348             final YangParser parser = parserFactory.createParser(parserConfig);
 349             for (final Entry<YangTextSchemaSource, YangIRSchemaSource> entry : parsed) {
 350                 final YangTextSchemaSource textSource = entry.getKey();
 351                 final YangIRSchemaSource astSource = entry.getValue();
 352                 parser.addSource(astSource);
 353
 354                 if (!astSource.getIdentifier().equals(textSource.getIdentifier())) {
 355                     // AST indicates a different source identifier, make sure we use that
 356                     sourcesInProject.add(YangTextSchemaSource.delegateForByteSource(astSource.getIdentifier(),
 357                         textSource));
 358                 } else {
 359                     sourcesInProject.add(textSource);
 360                 }
 361             }
 362
 363             final ProcessorModuleReactor reactor = new ProcessorModuleReactor(parser, sourcesInProject, dependencies);
 364             LOG.debug("Initialized reactor {} with {}", reactor, yangFilesInProject);
 365             return Optional.of(reactor);
 366         } catch (IOException | YangSyntaxErrorException | RuntimeException e) {
 367             // MojoExecutionException is thrown since execution cannot continue
 368             LOG.error("{} Unable to parse YANG files from {}", LOG_PREFIX, yangFilesRootDir, e);
 369             Throwable rootCause = Throwables.getRootCause(e);
 370             throw new MojoExecutionException(LOG_PREFIX + " Unable to parse YANG files from " + yangFilesRootDir,
 371                 rootCause);
 372         }
 373     }
 374
 375     private static ImmutableList<File> listFiles(final File root, final Collection<File> excludedFiles)
 376             throws IOException {
 377         if (!root.isDirectory()) {
 378             LOG.warn("{} YANG source directory {} not found. No code will be generated.", LOG_PREFIX, root);
 379             return ImmutableList.of();
 380         }
 381
 382         return Files.walk(root.toPath())
 383             .map(Path::toFile)
 384             .filter(File::isFile)
 385             .filter(f -> {
 386                 if (excludedFiles.contains(f)) {
 387                     LOG.info("{} YANG file excluded {}", LOG_PREFIX, f);
 388                     return false;
 389                 }
 390                 return true;
 391             })
 392             .filter(f -> f.getName().endsWith(YangConstants.RFC6020_YANG_FILE_EXTENSION))
 393             .collect(ImmutableList.toImmutableList());
 394     }
 395
 396     @VisibleForTesting
 397     static @NonNull Path stateFilePath(final String projectBuildDirectory) {
 398         // ${project.build.directory}/maven-status/yang-maven-plugin/execution.state
 399         return Path.of(projectBuildDirectory, "maven-status", "yang-maven-plugin", "execution.state");
 400     }
 401 }