yang/yang-repo-fs/src/main/java/org/opendaylight/yangtools/yang/model/repo/fs/FilesystemSchemaSourceCache.java

   1 /*
   2  * Copyright (c) 2014 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.yangtools.yang.model.repo.fs;
   9
  10 import static com.google.common.base.Preconditions.checkArgument;
  11 import static java.util.Objects.requireNonNull;
  12 import static org.opendaylight.yangtools.util.concurrent.FluentFutures.immediateFailedFluentFuture;
  13 import static org.opendaylight.yangtools.util.concurrent.FluentFutures.immediateFluentFuture;
  14
  15 import com.google.common.base.MoreObjects.ToStringHelper;
  16 import com.google.common.util.concurrent.FluentFuture;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.io.File;
  19 import java.io.FilenameFilter;
  20 import java.io.IOException;
  21 import java.io.InputStream;
  22 import java.nio.file.FileVisitResult;
  23 import java.nio.file.Files;
  24 import java.nio.file.Path;
  25 import java.nio.file.SimpleFileVisitor;
  26 import java.nio.file.StandardCopyOption;
  27 import java.nio.file.attribute.BasicFileAttributes;
  28 import java.time.format.DateTimeParseException;
  29 import java.util.ArrayList;
  30 import java.util.Collections;
  31 import java.util.List;
  32 import java.util.Map;
  33 import java.util.Optional;
  34 import java.util.TreeMap;
  35 import java.util.regex.Matcher;
  36 import java.util.regex.Pattern;
  37 import org.opendaylight.yangtools.yang.common.Revision;
  38 import org.opendaylight.yangtools.yang.model.repo.api.MissingSchemaSourceException;
  39 import org.opendaylight.yangtools.yang.model.repo.api.RevisionSourceIdentifier;
  40 import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceRepresentation;
  41 import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier;
  42 import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
  43 import org.opendaylight.yangtools.yang.model.repo.spi.AbstractSchemaSourceCache;
  44 import org.opendaylight.yangtools.yang.model.repo.spi.PotentialSchemaSource.Costs;
  45 import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48
  49 /**
  50  * Cache implementation that stores schemas in form of files under provided folder.
  51  */
  52 public final class FilesystemSchemaSourceCache<T extends SchemaSourceRepresentation>
  53         extends AbstractSchemaSourceCache<T> {
  54
  55     private static final Logger LOG = LoggerFactory.getLogger(FilesystemSchemaSourceCache.class);
  56
  57     // Init storage adapters
  58     private static final Map<Class<? extends SchemaSourceRepresentation>,
  59             StorageAdapter<? extends SchemaSourceRepresentation>> STORAGE_ADAPTERS = Collections.singletonMap(
  60                     YangTextSchemaSource.class, new YangTextSchemaStorageAdapter());
  61
  62     private static final Pattern CACHED_FILE_PATTERN =
  63             Pattern.compile("(?<moduleName>[^@]+)" + "(@(?<revision>" + Revision.STRING_FORMAT_PATTERN + "))?");
  64
  65     private final Class<T> representation;
  66     private final File storageDirectory;
  67
  68     public FilesystemSchemaSourceCache(
  69             final SchemaSourceRegistry consumer, final Class<T> representation, final File storageDirectory) {
  70         super(consumer, representation, Costs.LOCAL_IO);
  71         this.representation = representation;
  72         this.storageDirectory = requireNonNull(storageDirectory);
  73
  74         checkSupportedRepresentation(representation);
  75
  76         checkArgument(storageDirectory.mkdirs() || storageDirectory.isDirectory(),
  77                 "Unable to create cache directory at %s", storageDirectory);
  78         checkArgument(storageDirectory.canWrite());
  79         checkArgument(storageDirectory.canRead());
  80
  81         init();
  82     }
  83
  84     private static void checkSupportedRepresentation(final Class<? extends SchemaSourceRepresentation> representation) {
  85         for (final Class<? extends SchemaSourceRepresentation> supportedRepresentation : STORAGE_ADAPTERS.keySet()) {
  86             if (supportedRepresentation.isAssignableFrom(representation)) {
  87                 return;
  88             }
  89         }
  90
  91         throw new IllegalArgumentException(String.format(
  92                    "This cache does not support representation: %s, supported representations are: %s",
  93                    representation, STORAGE_ADAPTERS.keySet()));
  94     }
  95
  96     /**
  97      * Restore cache state.
  98      */
  99     private void init() {
 100
 101         final CachedModulesFileVisitor fileVisitor = new CachedModulesFileVisitor();
 102         try {
 103             Files.walkFileTree(storageDirectory.toPath(), fileVisitor);
 104         } catch (final IOException e) {
 105             LOG.warn("Unable to restore cache from {}. Starting with an empty cache", storageDirectory, e);
 106             return;
 107         }
 108
 109         fileVisitor.getCachedSchemas().stream().forEach(this::register);
 110     }
 111
 112     @Override
 113     public synchronized FluentFuture<? extends T> getSource(final SourceIdentifier sourceIdentifier) {
 114         final File file = sourceIdToFile(sourceIdentifier, storageDirectory);
 115         if (file.exists() && file.canRead()) {
 116             LOG.trace("Source {} found in cache as {}", sourceIdentifier, file);
 117             final SchemaSourceRepresentation restored = STORAGE_ADAPTERS.get(representation).restore(sourceIdentifier,
 118                     file);
 119             return immediateFluentFuture(representation.cast(restored));
 120         }
 121
 122         LOG.debug("Source {} not found in cache as {}", sourceIdentifier, file);
 123         return immediateFailedFluentFuture(new MissingSchemaSourceException("Source not found", sourceIdentifier));
 124     }
 125
 126     @Override
 127     protected synchronized void offer(final T source) {
 128         LOG.trace("Source {} offered to cache", source.getIdentifier());
 129         final File file = sourceIdToFile(source);
 130         if (file.exists()) {
 131             LOG.debug("Source {} already in cache as {}", source.getIdentifier(), file);
 132             return;
 133         }
 134
 135         storeSource(file, source);
 136         register(source.getIdentifier());
 137         LOG.trace("Source {} stored in cache as {}", source.getIdentifier(), file);
 138     }
 139
 140     private File sourceIdToFile(final T source) {
 141         return sourceIdToFile(source.getIdentifier(), storageDirectory);
 142     }
 143
 144     static File sourceIdToFile(final SourceIdentifier identifier, final File storageDirectory) {
 145         final Optional<Revision> rev = identifier.getRevision();
 146         final File file;
 147         if (rev.isEmpty()) {
 148             // FIXME: this does not look right
 149             file = findFileWithNewestRev(identifier, storageDirectory);
 150         } else {
 151             file = new File(storageDirectory, identifier.toYangFilename());
 152         }
 153         return file;
 154     }
 155
 156     @SuppressFBWarnings(value = "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE",
 157             justification = "listFiles is analyzed to return null")
 158     private static File findFileWithNewestRev(final SourceIdentifier identifier, final File storageDirectory) {
 159         File[] files = storageDirectory.listFiles(new FilenameFilter() {
 160             final Pattern pat = Pattern.compile(Pattern.quote(identifier.getName())
 161                     + "(\\.yang|@\\d\\d\\d\\d-\\d\\d-\\d\\d.yang)");
 162
 163             @Override
 164             public boolean accept(final File dir, final String name) {
 165                 return pat.matcher(name).matches();
 166             }
 167         });
 168
 169         if (files.length == 0) {
 170             return new File(storageDirectory, identifier.toYangFilename());
 171         }
 172         if (files.length == 1) {
 173             return files[0];
 174         }
 175
 176         File file = null;
 177         TreeMap<Optional<Revision>, File> map = new TreeMap<>(Revision::compare);
 178         for (File sorted : files) {
 179             String fileName = sorted.getName();
 180             Matcher match = Revision.STRING_FORMAT_PATTERN.matcher(fileName);
 181             if (match.find()) {
 182                 String revStr = match.group();
 183                 Revision rev;
 184                 try {
 185                     rev = Revision.of(revStr);
 186                 } catch (final DateTimeParseException e) {
 187                     LOG.info("Unable to parse date from yang file name {}, falling back to not-present", fileName, e);
 188                     rev = null;
 189                 }
 190
 191                 map.put(Optional.ofNullable(rev), sorted);
 192
 193             } else {
 194                 map.put(Optional.empty(), sorted);
 195             }
 196         }
 197         file = map.lastEntry().getValue();
 198
 199         return file;
 200     }
 201
 202     private void storeSource(final File file, final T schemaRepresentation) {
 203         STORAGE_ADAPTERS.get(representation).store(file, schemaRepresentation);
 204     }
 205
 206     private abstract static class StorageAdapter<T extends SchemaSourceRepresentation> {
 207
 208         private final Class<T> supportedType;
 209
 210         protected StorageAdapter(final Class<T> supportedType) {
 211             this.supportedType = supportedType;
 212         }
 213
 214         void store(final File file, final SchemaSourceRepresentation schemaSourceRepresentation) {
 215             checkArgument(supportedType.isAssignableFrom(schemaSourceRepresentation.getClass()),
 216                     "Cannot store schema source %s, this adapter only supports %s", schemaSourceRepresentation,
 217                     supportedType);
 218
 219             storeAsType(file, supportedType.cast(schemaSourceRepresentation));
 220         }
 221
 222         protected abstract void storeAsType(File file, T cast);
 223
 224         public T restore(final SourceIdentifier sourceIdentifier, final File cachedSource) {
 225             checkArgument(cachedSource.isFile());
 226             checkArgument(cachedSource.exists());
 227             checkArgument(cachedSource.canRead());
 228             return restoreAsType(sourceIdentifier, cachedSource);
 229         }
 230
 231         protected abstract T restoreAsType(SourceIdentifier sourceIdentifier, File cachedSource);
 232     }
 233
 234     private static final class YangTextSchemaStorageAdapter extends StorageAdapter<YangTextSchemaSource> {
 235
 236         protected YangTextSchemaStorageAdapter() {
 237             super(YangTextSchemaSource.class);
 238         }
 239
 240         @Override
 241         @SuppressFBWarnings(value = "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
 242             justification = "https://github.com/spotbugs/spotbugs/issues/600")
 243         protected void storeAsType(final File file, final YangTextSchemaSource cast) {
 244             try (InputStream castStream = cast.openStream()) {
 245                 Files.copy(castStream, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
 246             } catch (final IOException e) {
 247                 throw new IllegalStateException("Cannot store schema source " + cast.getIdentifier() + " to " + file,
 248                         e);
 249             }
 250         }
 251
 252         @Override
 253         public YangTextSchemaSource restoreAsType(final SourceIdentifier sourceIdentifier, final File cachedSource) {
 254             return new YangTextSchemaSource(sourceIdentifier) {
 255
 256                 @Override
 257                 protected ToStringHelper addToStringAttributes(final ToStringHelper toStringHelper) {
 258                     return toStringHelper;
 259                 }
 260
 261                 @Override
 262                 public InputStream openStream() throws IOException {
 263                     return Files.newInputStream(cachedSource.toPath());
 264                 }
 265             };
 266         }
 267     }
 268
 269     private static final class CachedModulesFileVisitor extends SimpleFileVisitor<Path> {
 270         private final List<SourceIdentifier> cachedSchemas = new ArrayList<>();
 271
 272         @Override
 273         public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
 274             final FileVisitResult fileVisitResult = super.visitFile(file, attrs);
 275             String fileName = file.toFile().getName();
 276             fileName = com.google.common.io.Files.getNameWithoutExtension(fileName);
 277
 278             final Optional<SourceIdentifier> si = getSourceIdentifier(fileName);
 279             if (si.isPresent()) {
 280                 LOG.trace("Restoring cached file {} as {}", file, si.get());
 281                 cachedSchemas.add(si.get());
 282             } else {
 283                 LOG.debug("Skipping cached file {}, cannot restore source identifier from filename: {},"
 284                         + " does not match {}", file, fileName, CACHED_FILE_PATTERN);
 285             }
 286             return fileVisitResult;
 287         }
 288
 289         private static Optional<SourceIdentifier> getSourceIdentifier(final String fileName) {
 290             final Matcher matcher = CACHED_FILE_PATTERN.matcher(fileName);
 291             if (matcher.matches()) {
 292                 final String moduleName = matcher.group("moduleName");
 293                 final String revision = matcher.group("revision");
 294                 return Optional.of(RevisionSourceIdentifier.create(moduleName, Revision.ofNullable(revision)));
 295             }
 296             return Optional.empty();
 297         }
 298
 299         @Override
 300         public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException {
 301             LOG.warn("Unable to restore cached file {}. Ignoring", file, exc);
 302             return FileVisitResult.CONTINUE;
 303         }
 304
 305         public List<SourceIdentifier> getCachedSchemas() {
 306             return cachedSchemas;
 307         }
 308     }
 309 }