opendaylight/sal/yang-prototype/code-generator/maven-yang-plugin/src/main/java/org/opendaylight/controller/yang2sources/plugin/YangToSourcesMojo.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.yang2sources.plugin;
   9
  10 import java.io.Closeable;
  11 import java.io.File;
  12 import java.io.FileInputStream;
  13 import java.io.FileNotFoundException;
  14 import java.io.FileOutputStream;
  15 import java.io.IOException;
  16 import java.io.InputStream;
  17 import java.util.ArrayList;
  18 import java.util.Collection;
  19 import java.util.Collections;
  20 import java.util.Enumeration;
  21 import java.util.HashSet;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.zip.ZipEntry;
  26 import java.util.zip.ZipFile;
  27
  28 import org.apache.commons.io.IOUtils;
  29 import org.apache.maven.model.Resource;
  30 import org.apache.maven.plugin.AbstractMojo;
  31 import org.apache.maven.plugin.MojoExecutionException;
  32 import org.apache.maven.plugin.MojoFailureException;
  33 import org.apache.maven.plugins.annotations.LifecyclePhase;
  34 import org.apache.maven.plugins.annotations.Mojo;
  35 import org.apache.maven.plugins.annotations.Parameter;
  36 import org.apache.maven.plugins.annotations.ResolutionScope;
  37 import org.apache.maven.project.MavenProject;
  38 import org.opendaylight.controller.yang.model.api.Module;
  39 import org.opendaylight.controller.yang.model.api.SchemaContext;
  40 import org.opendaylight.controller.yang.model.parser.api.YangModelParser;
  41 import org.opendaylight.controller.yang.parser.impl.YangParserImpl;
  42 import org.opendaylight.controller.yang2sources.plugin.ConfigArg.CodeGeneratorArg;
  43 import org.opendaylight.controller.yang2sources.plugin.ConfigArg.ResourceProviderArg;
  44 import org.opendaylight.controller.yang2sources.spi.CodeGenerator;
  45 import org.opendaylight.controller.yang2sources.spi.ResourceGenerator;
  46
  47 import com.google.common.annotations.VisibleForTesting;
  48 import com.google.common.collect.Lists;
  49 import com.google.common.collect.Maps;
  50 import com.google.common.io.Files;
  51
  52 /**
  53  * Generate sources from yang files using user provided set of
  54  * {@link CodeGenerator}s. Steps of this process:
  55  * <ol>
  56  * <li>List yang files from {@link #yangFilesRootDir}</li>
  57  * <li>Process yang files using {@link YangModelParserImpl}</li>
  58  * <li>For each {@link CodeGenerator} from {@link #codeGenerators}:</li>
  59  * <ol>
  60  * <li>Instantiate using default constructor</li>
  61  * <li>Call {@link CodeGenerator#generateSources(SchemaContext, File)}</li>
  62  * </ol>
  63  * </ol>
  64  */
  65 @Mojo(name = "generate-sources", defaultPhase = LifecyclePhase.GENERATE_SOURCES, requiresDependencyResolution = ResolutionScope.COMPILE, requiresProject = true)
  66 public final class YangToSourcesMojo extends AbstractMojo {
  67     private static final String LOG_PREFIX = "yang-to-sources:";
  68     private static final String INPUT_RESOURCE_DIR = "META-INF/yang/";
  69     private static final String OUTPUT_RESOURCE_DIR = "/target/external-resources/";
  70
  71     /**
  72      * Classes implementing {@link CodeGenerator} interface. An instance will be
  73      * created out of every class using default constructor. Method {@link
  74      * CodeGenerator#generateSources(SchemaContext, File, Set<String>
  75      * yangModulesNames)} will be called on every instance.
  76      */
  77     @Parameter(required = true)
  78     private CodeGeneratorArg[] codeGenerators;
  79
  80     /**
  81      * Source directory that will be recursively searched for yang files (ending
  82      * with .yang suffix).
  83      */
  84     @Parameter(required = true)
  85     private String yangFilesRootDir;
  86
  87     /**
  88      * Classes implementing {@link ResourceGenerator} interface. An instance
  89      * will be created out of every class using default constructor. Method
  90      * {@link ResourceGenerator#generateResourceFiles(Collection, File)} will be
  91      * called on every instance.
  92      */
  93     @Parameter(required = true)
  94     private ResourceProviderArg[] resourceProviders;
  95
  96     @Parameter(property = "project", required = true, readonly = true)
  97     protected MavenProject project;
  98
  99     private transient final YangModelParser parser;
 100
 101     @VisibleForTesting
 102     YangToSourcesMojo(ResourceProviderArg[] resourceProviderArgs,
 103             CodeGeneratorArg[] codeGeneratorArgs, YangModelParser parser,
 104             String yangFilesRootDir) {
 105         super();
 106         this.resourceProviders = resourceProviderArgs;
 107         this.codeGenerators = codeGeneratorArgs;
 108         this.yangFilesRootDir = yangFilesRootDir;
 109         this.parser = parser;
 110     }
 111
 112     public YangToSourcesMojo() {
 113         super();
 114         parser = new YangParserImpl();
 115     }
 116
 117     @Override
 118     public void execute() throws MojoExecutionException, MojoFailureException {
 119         ContextHolder context = processYang();
 120         generateSources(context);
 121         generateResources();
 122
 123         closeResources();
 124     }
 125
 126     /**
 127      * Generate {@link SchemaContext} with {@link YangModelParserImpl}
 128      */
 129     private ContextHolder processYang() throws MojoExecutionException {
 130         try {
 131             List<InputStream> yangFiles = Util
 132                     .listFilesAsStream(yangFilesRootDir);
 133             Set<Module> yangModules = parser
 134                     .parseYangModelsFromStreams(yangFiles);
 135
 136             Set<String> yangModulesNames = new HashSet<String>();
 137             for (Module module : yangModules) {
 138                 yangModulesNames.add(module.getName());
 139             }
 140
 141             List<InputStream> yangFilesFromDependencies = getFilesFromDependenciesAsStream();
 142             Set<Module> yangModulesFromDependencies = parser
 143                     .parseYangModelsFromStreams(yangFilesFromDependencies);
 144
 145             Set<Module> parsedYang = new HashSet<Module>(yangModules);
 146             parsedYang.addAll(yangModulesFromDependencies);
 147
 148             if (yangFiles.isEmpty() && yangFilesFromDependencies.isEmpty()) {
 149                 getLog().warn(
 150                         Util.message(
 151                                 "No %s file found in %s or in dependencies",
 152                                 LOG_PREFIX, Util.YANG_SUFFIX, yangFilesRootDir));
 153                 Set<String> names = Collections.emptySet();
 154                 return new ContextHolder(null, names);
 155             }
 156
 157             SchemaContext resolveSchemaContext = parser
 158                     .resolveSchemaContext(parsedYang);
 159             getLog().info(
 160                     Util.message("%s files parsed from %s", LOG_PREFIX,
 161                             Util.YANG_SUFFIX, yangFiles));
 162             return new ContextHolder(resolveSchemaContext, yangModulesNames);
 163
 164             // MojoExecutionException is thrown since execution cannot continue
 165         } catch (Exception e) {
 166             String message = Util.message("Unable to parse %s files from %s",
 167                     LOG_PREFIX, Util.YANG_SUFFIX, yangFilesRootDir);
 168             getLog().error(message, e);
 169             throw new MojoExecutionException(message, e);
 170         }
 171     }
 172
 173     private void generateResources() throws MojoExecutionException,
 174             MojoFailureException {
 175         if (resourceProviders.length == 0) {
 176             getLog().warn(
 177                     Util.message("No resource provider classes provided",
 178                             LOG_PREFIX));
 179             return;
 180         }
 181
 182         Resource res = new Resource();
 183         String baseDirName = project.getBasedir().getAbsolutePath();
 184         res.setDirectory(baseDirName + OUTPUT_RESOURCE_DIR);
 185         res.setTargetPath(INPUT_RESOURCE_DIR);
 186         project.addResource(res);
 187
 188         Map<String, String> thrown = Maps.newHashMap();
 189
 190         Collection<File> yangFiles = new ArrayList<File>();
 191
 192         // load files from yang root
 193         yangFiles.addAll(getFilesFromYangRoot());
 194
 195         // load files from dependencies
 196         Collection<File> filesFromDependencies = getFilesFromDependencies();
 197         yangFiles.addAll(filesFromDependencies);
 198
 199         for (ResourceProviderArg resourceProvider : resourceProviders) {
 200             try {
 201                 provideResourcesWithOneProvider(yangFiles, resourceProvider);
 202             } catch (Exception e) {
 203                 // try other generators, exception will be thrown after
 204                 getLog().error(
 205                         Util.message(
 206                                 "Unable to provide resources with %s resource provider",
 207                                 LOG_PREFIX,
 208                                 resourceProvider.getResourceProviderClass()), e);
 209                 thrown.put(resourceProvider.getResourceProviderClass(), e
 210                         .getClass().getCanonicalName());
 211             }
 212         }
 213
 214         if (!thrown.isEmpty()) {
 215             String message = Util
 216                     .message(
 217                             "One or more code resource provider failed, including failed list(resourceProviderClass=exception) %s",
 218                             LOG_PREFIX, thrown.toString());
 219             getLog().error(message);
 220             throw new MojoFailureException(message);
 221         }
 222     }
 223
 224     private Collection<File> getFilesFromYangRoot() {
 225         Collection<File> yangFilesLoaded = null;
 226
 227         File rootDir = new File(yangFilesRootDir);
 228         try {
 229             if (rootDir.isAbsolute()) {
 230                 yangFilesLoaded = Util.listFiles(yangFilesRootDir);
 231             } else {
 232                 String path = project.getBasedir().getAbsolutePath()
 233                         + File.separator + yangFilesRootDir;
 234                 yangFilesLoaded = Util.listFiles(path);
 235             }
 236         } catch (FileNotFoundException e) {
 237             getLog().warn(
 238                     "yangFilesRootDir[" + rootDir.getAbsolutePath()
 239                             + "] does not exists.");
 240             yangFilesLoaded = new ArrayList<File>();
 241         }
 242
 243         Collection<File> yangFiles = new ArrayList<File>(yangFilesLoaded);
 244
 245         try {
 246             for (File yangFile : yangFilesLoaded) {
 247                 InputStream is = new FileInputStream(yangFile);
 248                 yangFiles.add(createFileFromStream(is,
 249                         project.getBasedir().getAbsolutePath()
 250                                 + OUTPUT_RESOURCE_DIR + yangFile.getName()));
 251                 resources.add(is);
 252             }
 253         } catch (IOException e) {
 254             getLog().warn("Exception while loading yang files.", e);
 255         }
 256         return yangFiles;
 257     }
 258
 259     private Collection<File> getFilesFromDependencies() {
 260         Collection<File> yangFiles = new ArrayList<File>();
 261
 262         try {
 263             List<File> filesOnCp = Util.getClassPath(project);
 264             List<String> filter = Lists.newArrayList(".yang");
 265             for (File file : filesOnCp) {
 266                 ZipFile zip = new ZipFile(file);
 267                 Enumeration<? extends ZipEntry> entries = zip.entries();
 268
 269                 while (entries.hasMoreElements()) {
 270                     ZipEntry entry = entries.nextElement();
 271                     String entryName = entry.getName();
 272                     if (entryName.startsWith(INPUT_RESOURCE_DIR)) {
 273                         if (entry.isDirectory()) {
 274                             continue;
 275                         }
 276                         if (!Util.acceptedFilter(entryName, filter)) {
 277                             continue;
 278                         }
 279                         InputStream entryStream = zip.getInputStream(entry);
 280                         String newEntryName = entryName
 281                                 .substring(INPUT_RESOURCE_DIR.length());
 282                         File tmp = Files.createTempDir();
 283                         File f = createFileFromStream(entryStream,
 284                                 tmp.getAbsolutePath() + newEntryName);
 285                         yangFiles.add(f);
 286
 287                         resources.add(entryStream);
 288                     }
 289                 }
 290
 291                 resources.add(zip);
 292             }
 293         } catch (Exception e) {
 294             getLog().warn("Exception while loading external yang files.", e);
 295         }
 296         return yangFiles;
 297     }
 298
 299     private File createFileFromStream(InputStream is, String absoluteName)
 300             throws IOException {
 301         File f = new File(absoluteName);
 302         if (!f.exists()) {
 303             f.getParentFile().mkdirs();
 304         }
 305         f.createNewFile();
 306
 307         FileOutputStream fos = new FileOutputStream(f);
 308         IOUtils.copy(is, fos);
 309         return f;
 310     }
 311
 312     /**
 313      * Instantiate provider from class and call required method
 314      */
 315     private void provideResourcesWithOneProvider(Collection<File> yangFiles,
 316             ResourceProviderArg resourceProvider)
 317             throws ClassNotFoundException, InstantiationException,
 318             IllegalAccessException {
 319
 320         resourceProvider.check();
 321
 322         ResourceGenerator g = Util.getInstance(
 323                 resourceProvider.getResourceProviderClass(),
 324                 ResourceGenerator.class);
 325         getLog().info(
 326                 Util.message("Resource provider instantiated from %s",
 327                         LOG_PREFIX, resourceProvider.getResourceProviderClass()));
 328
 329         g.generateResourceFiles(yangFiles, resourceProvider.getOutputBaseDir());
 330         getLog().info(
 331                 Util.message("Resource provider %s call successful",
 332                         LOG_PREFIX, resourceProvider.getResourceProviderClass()));
 333     }
 334
 335     /**
 336      * Call generate on every generator from plugin configuration
 337      */
 338     private void generateSources(ContextHolder context)
 339             throws MojoFailureException {
 340         if (codeGenerators.length == 0) {
 341             getLog().warn(
 342                     Util.message("No code generators provided", LOG_PREFIX));
 343             return;
 344         }
 345
 346         Map<String, String> thrown = Maps.newHashMap();
 347         for (CodeGeneratorArg codeGenerator : codeGenerators) {
 348             try {
 349                 generateSourcesWithOneGenerator(context, codeGenerator);
 350             } catch (Exception e) {
 351                 // try other generators, exception will be thrown after
 352                 getLog().error(
 353                         Util.message(
 354                                 "Unable to generate sources with %s generator",
 355                                 LOG_PREFIX,
 356                                 codeGenerator.getCodeGeneratorClass()), e);
 357                 thrown.put(codeGenerator.getCodeGeneratorClass(), e.getClass()
 358                         .getCanonicalName());
 359             }
 360         }
 361
 362         if (!thrown.isEmpty()) {
 363             String message = Util
 364                     .message(
 365                             "One or more code generators failed, including failed list(generatorClass=exception) %s",
 366                             LOG_PREFIX, thrown.toString());
 367             getLog().error(message);
 368             throw new MojoFailureException(message);
 369         }
 370     }
 371
 372     /**
 373      * Instantiate generator from class and call required method
 374      */
 375     private void generateSourcesWithOneGenerator(ContextHolder context,
 376             CodeGeneratorArg codeGeneratorCfg) throws ClassNotFoundException,
 377             InstantiationException, IllegalAccessException, IOException {
 378
 379         codeGeneratorCfg.check();
 380
 381         CodeGenerator g = Util.getInstance(
 382                 codeGeneratorCfg.getCodeGeneratorClass(), CodeGenerator.class);
 383         getLog().info(
 384                 Util.message("Code generator instantiated from %s", LOG_PREFIX,
 385                         codeGeneratorCfg.getCodeGeneratorClass()));
 386
 387         File outputDir = codeGeneratorCfg.getOutputBaseDir();
 388         if (project != null && outputDir != null) {
 389             project.addCompileSourceRoot(outputDir.getPath());
 390         }
 391         Collection<File> generated = g.generateSources(context.getContext(),
 392                 outputDir, context.getYangModulesNames());
 393         getLog().info(
 394                 Util.message("Sources generated by %s: %s", LOG_PREFIX,
 395                         codeGeneratorCfg.getCodeGeneratorClass(), generated));
 396     }
 397
 398     /**
 399      * Collection of resources which should be closed after use.
 400      */
 401     private final List<Closeable> resources = new ArrayList<Closeable>();
 402
 403     /**
 404      * Search for yang files in dependent projects.
 405      *
 406      * @return files found as List of InputStream
 407      */
 408     private List<InputStream> getFilesFromDependenciesAsStream() {
 409         final List<InputStream> yangsFromDependencies = new ArrayList<InputStream>();
 410         try {
 411             List<File> filesOnCp = Util.getClassPath(project);
 412
 413             List<String> filter = Lists.newArrayList(".yang");
 414             for (File file : filesOnCp) {
 415                 ZipFile zip = new ZipFile(file);
 416                 Enumeration<? extends ZipEntry> entries = zip.entries();
 417                 while (entries.hasMoreElements()) {
 418                     ZipEntry entry = entries.nextElement();
 419                     String entryName = entry.getName();
 420
 421                     if (entryName.startsWith(INPUT_RESOURCE_DIR)) {
 422                         if (entry.isDirectory()) {
 423                             continue;
 424                         }
 425                         if (!Util.acceptedFilter(entryName, filter)) {
 426                             continue;
 427                         }
 428
 429                         InputStream entryStream = zip.getInputStream(entry);
 430                         yangsFromDependencies.add(entryStream);
 431                         resources.add(entryStream);
 432                     }
 433
 434                 }
 435                 resources.add(zip);
 436             }
 437         } catch (Exception e) {
 438             getLog().warn("Exception while searching yangs in dependencies", e);
 439         }
 440         return yangsFromDependencies;
 441     }
 442
 443     /**
 444      * Internal utility method for closing open resources.
 445      */
 446     private void closeResources() {
 447         for (Closeable resource : resources) {
 448             try {
 449                 resource.close();
 450             } catch (IOException e) {
 451                 getLog().warn("Failed to close resources: " + resource, e);
 452             }
 453         }
 454     }
 455
 456     private class ContextHolder {
 457         private final SchemaContext context;
 458         private final Set<String> yangModulesNames;
 459
 460         private ContextHolder(SchemaContext context,
 461                 Set<String> yangModulesNames) {
 462             this.context = context;
 463             this.yangModulesNames = yangModulesNames;
 464         }
 465
 466         public SchemaContext getContext() {
 467             return context;
 468         }
 469
 470         public Set<String> getYangModulesNames() {
 471             return yangModulesNames;
 472         }
 473     }
 474
 475 }