From a6fb0b3903e8630df18b7c63a116bfd06a716bdf Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Tue, 7 Jan 2020 21:35:22 +0100 Subject: [PATCH] Generate binding files in parallel Binding-generated files are standalone in that the process of generating its source does not have any other side effects. Improve the performance by generating these files in parallel using the common ForkJoin pool -- thus taking advantage of all available cores. Change-Id: I4ee9efea0371fe9574721701ea9674793209cf24 Signed-off-by: Robert Varga --- .../api/gen/plugin/CodeGeneratorImpl.java | 113 +++++++++++++++--- 1 file changed, 99 insertions(+), 14 deletions(-) diff --git a/binding/maven-sal-api-gen-plugin/src/main/java/org/opendaylight/mdsal/binding/maven/api/gen/plugin/CodeGeneratorImpl.java b/binding/maven-sal-api-gen-plugin/src/main/java/org/opendaylight/mdsal/binding/maven/api/gen/plugin/CodeGeneratorImpl.java index 5a0aa38ffc..5aa8d58489 100644 --- a/binding/maven-sal-api-gen-plugin/src/main/java/org/opendaylight/mdsal/binding/maven/api/gen/plugin/CodeGeneratorImpl.java +++ b/binding/maven-sal-api-gen-plugin/src/main/java/org/opendaylight/mdsal/binding/maven/api/gen/plugin/CodeGeneratorImpl.java @@ -10,11 +10,18 @@ package org.opendaylight.mdsal.binding.maven.api.gen.plugin; import static java.util.Objects.requireNonNull; import com.google.common.base.Joiner; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.MultimapBuilder; import com.google.common.collect.Table; import com.google.common.collect.Table.Cell; -import com.google.common.io.Files; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; @@ -22,14 +29,21 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.apache.maven.project.MavenProject; import org.opendaylight.mdsal.binding.generator.impl.BindingGeneratorImpl; import org.opendaylight.mdsal.binding.java.api.generator.GeneratorJavaFile; @@ -67,7 +81,11 @@ public final class CodeGeneratorImpl implements BasicCodeGenerator, BuildContext outputBaseDir = outputDir == null ? getDefaultOutputBaseDir() : outputDir; + // Step one: determine binding types which we are generating + final Stopwatch sw = Stopwatch.createStarted(); final List types = new BindingGeneratorImpl().generateTypes(context, yangModules); + LOG.info("Found {} Binding types in {}", types.size(), sw); + final GeneratorJavaFile generator = new GeneratorJavaFile(types); File persistentSourcesDir = null; @@ -88,7 +106,9 @@ public final class CodeGeneratorImpl implements BasicCodeGenerator, BuildContext final Table> generatedFiles = generator.generateFileContent( ignoreDuplicateFiles); - final List result = new ArrayList<>(generatedFiles.size()); + + // Step two: create generation tasks for each target file and group them by parent directory + final ListMultimap dirs = MultimapBuilder.hashKeys().arrayListValues().build(); for (Cell> cell : generatedFiles.cellSet()) { final File target; switch (cell.getRowKey()) { @@ -106,21 +126,62 @@ public final class CodeGeneratorImpl implements BasicCodeGenerator, BuildContext throw new IllegalStateException("Unsupported file type in " + cell); } - Files.createParentDirs(target); - try (OutputStream stream = buildContext.newFileOutputStream(target)) { - try (Writer fw = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) { - try (BufferedWriter bw = new BufferedWriter(fw)) { - bw.write(cell.getValue().get()); - } - } catch (IOException e) { - LOG.error("Failed to write generate output into {}", target.getPath(), e); - throw e; - } - } + dirs.put(target.getParentFile().toPath(), new GenerationTask(buildContext, target, cell.getValue())); + } + LOG.info("Generating {} Binding source files into {} directories", dirs.size(), dirs.keySet().size()); + + // Step three: wrap common FJ pool, submit parent directory creation tasks and wait for them to complete + sw.reset().start(); + final ListeningExecutorService service = MoreExecutors.listeningDecorator(ForkJoinPool.commonPool()); + final List> parentFutures = new ArrayList<>(dirs.keySet().size()); + for (Entry> entry : dirs.asMap().entrySet()) { + parentFutures.add(service.submit(() -> { + Files.createDirectories(entry.getKey()); + return null; + })); + } - result.add(target); + try { + Futures.whenAllComplete(parentFutures).call(() -> { + for (ListenableFuture future : parentFutures) { + Futures.getDone(future); + } + return null; + }, MoreExecutors.directExecutor()).get(); + } catch (InterruptedException e) { + throw new IOException("Interrupted while creating parent directories", e); + } catch (ExecutionException e) { + LOG.debug("Failed to create parent directories", e); + Throwables.throwIfInstanceOf(e.getCause(), IOException.class); + throw new IOException("Failed to create parent directories", e); + } + LOG.debug("Parent directories created in {}", sw); + + // Step four: submit all code generation tasks and wait for them to complete + sw.reset().start(); + final List> futureFiles = dirs.values().stream() + .map(service::submit) + .collect(Collectors.toList()); + + final List result; + try { + result = Futures.whenAllComplete(futureFiles).call(() -> { + List ret = new ArrayList<>(futureFiles.size()); + for (ListenableFuture future : futureFiles) { + ret.add(Futures.getDone(future)); + } + return ret; + }, MoreExecutors.directExecutor()).get(); + } catch (InterruptedException e) { + throw new IOException("Interrupted while generating files", e); + } catch (ExecutionException e) { + LOG.error("Failed to create generated files", e); + Throwables.throwIfInstanceOf(e.getCause(), IOException.class); + throw new IOException("Failed to create generated files", e); } + LOG.debug("{} Binding source type files generated in {}", result.size(), sw); + // Step five: generate auxiliary files result.addAll(generateModuleInfos(outputBaseDir, yangModules, context, moduleResourcePathResolver)); return result; } @@ -240,4 +301,28 @@ public final class CodeGeneratorImpl implements BasicCodeGenerator, BuildContext } return file; } + + private static final class GenerationTask implements Callable { + private final BuildContext buildContext; + private final Supplier contentSupplier; + private final File target; + + GenerationTask(final BuildContext buildContext, final File target, final Supplier contentSupplier) { + this.buildContext = requireNonNull(buildContext); + this.target = requireNonNull(target); + this.contentSupplier = requireNonNull(contentSupplier); + } + + @Override + public File call() throws IOException { + try (OutputStream stream = buildContext.newFileOutputStream(target)) { + try (Writer fw = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) { + try (BufferedWriter bw = new BufferedWriter(fw)) { + bw.write(contentSupplier.get()); + } + } + } + return target; + } + } } -- 2.36.6