From 24615339eebb94b10b8b922f62544a96a38e7386 Mon Sep 17 00:00:00 2001 From: Maros Marsalek Date: Wed, 6 Aug 2014 10:42:22 +0200 Subject: [PATCH] BUG-997 Implement Filesystem source cache for schema repository Use utility methods for storing files from legacy Filesystem cache + Add sorting by Cost to schema repository for multiple providers of the same source Change-Id: I9a30240042e34fe7898966467b503eee85166887 Signed-off-by: Maros Marsalek --- yang/yang-model-util/pom.xml | 10 + .../repo/util/AbstractSchemaRepository.java | 51 +++- .../util/FilesystemSchemaSourceCache.java | 252 ++++++++++++++++++ .../repo/FilesystemSchemaCachingProvider.java | 10 +- .../util/FilesystemSchemaSourceCacheTest.java | 164 ++++++++++++ .../repo/SharedSchemaRepositoryTest.java | 164 +++++++++++- 6 files changed, 632 insertions(+), 19 deletions(-) create mode 100644 yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCache.java create mode 100644 yang/yang-model-util/src/test/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCacheTest.java diff --git a/yang/yang-model-util/pom.xml b/yang/yang-model-util/pom.xml index c4c5a8720f..a2bb89c602 100644 --- a/yang/yang-model-util/pom.xml +++ b/yang/yang-model-util/pom.xml @@ -33,6 +33,16 @@ junit junit + test + + + org.mockito + mockito-core + + + ch.qos.logback + logback-classic + test diff --git a/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/AbstractSchemaRepository.java b/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/AbstractSchemaRepository.java index 88bcb300f8..4b530e7dfb 100644 --- a/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/AbstractSchemaRepository.java +++ b/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/AbstractSchemaRepository.java @@ -8,22 +8,23 @@ package org.opendaylight.yangtools.yang.model.repo.util; import com.google.common.annotations.Beta; -import com.google.common.collect.HashMultimap; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.CheckedFuture; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.FutureFallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.Map; - import javax.annotation.concurrent.GuardedBy; - import org.opendaylight.yangtools.util.concurrent.ExceptionMapper; import org.opendaylight.yangtools.util.concurrent.ReflectiveExceptionMapper; import org.opendaylight.yangtools.yang.model.repo.api.MissingSchemaSourceException; @@ -56,7 +57,7 @@ public abstract class AbstractSchemaRepository implements SchemaRepository, Sche * a specific representation of a source. */ @GuardedBy("this") - private final Map, AbstractSchemaSourceRegistration>> sources = new HashMap<>(); + private final Map, AbstractSchemaSourceRegistration>> sources = new HashMap<>(); /* * Schema source listeners. @@ -69,6 +70,7 @@ public abstract class AbstractSchemaRepository implements SchemaRepository, Sche @SuppressWarnings("unchecked") final CheckedFuture f = ((SchemaSourceProvider)reg.getProvider()).getSource(id); + return Futures.makeChecked(Futures.withFallback(f, new FutureFallback() { @Override public ListenableFuture create(final Throwable t) throws SchemaSourceException { @@ -85,24 +87,44 @@ public abstract class AbstractSchemaRepository implements SchemaRepository, Sche @Override public CheckedFuture getSchemaSource(final SourceIdentifier id, final Class representation) { - final Multimap, AbstractSchemaSourceRegistration> srcs = sources.get(id); + final ListMultimap, AbstractSchemaSourceRegistration> srcs = sources.get(id); if (srcs == null) { return Futures.immediateFailedCheckedFuture(new MissingSchemaSourceException("No providers registered for source" + id)); } - final Iterator> regs = srcs.get(representation).iterator(); + // TODO, remove and make sources keep sorted multimap (e.g. ArrayListMultimap with SortedLists) + final ArrayList> sortedSchemaSourceRegistrations = Lists.newArrayList(srcs.get(representation)); + Collections.sort(sortedSchemaSourceRegistrations, SchemaProviderCostComparator.INSTANCE); + + final Iterator> regs = sortedSchemaSourceRegistrations.iterator(); if (!regs.hasNext()) { return Futures.immediateFailedCheckedFuture( new MissingSchemaSourceException("No providers for source " + id + " representation " + representation + " available")); } - return fetchSource(id, regs); + CheckedFuture fetchSourceFuture = fetchSource(id, regs); + // Add callback to notify cache listeners about encountered schema + Futures.addCallback(fetchSourceFuture, new FutureCallback() { + @Override + public void onSuccess(final T result) { + for (final SchemaListenerRegistration listener : listeners) { + listener.getInstance().schemaSourceEncountered(result); + } + } + + @Override + public void onFailure(final Throwable t) { + LOG.trace("Skipping notification for encountered source {}, fetching source failed", id, t); + } + }); + + return fetchSourceFuture; } private synchronized void addSource(final PotentialSchemaSource source, final AbstractSchemaSourceRegistration reg) { - Multimap, AbstractSchemaSourceRegistration> m = sources.get(source.getSourceIdentifier()); + ListMultimap, AbstractSchemaSourceRegistration> m = sources.get(source.getSourceIdentifier()); if (m == null) { - m = HashMultimap.create(); + m = ArrayListMultimap.create(); sources.put(source.getSourceIdentifier(), m); } @@ -166,4 +188,13 @@ public abstract class AbstractSchemaRepository implements SchemaRepository, Sche } return ret; } + + private static class SchemaProviderCostComparator implements Comparator> { + public static final SchemaProviderCostComparator INSTANCE = new SchemaProviderCostComparator(); + + @Override + public int compare(final AbstractSchemaSourceRegistration o1, final AbstractSchemaSourceRegistration o2) { + return o1.getInstance().getCost() - o2.getInstance().getCost(); + } + } } diff --git a/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCache.java b/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCache.java new file mode 100644 index 0000000000..8fc07a1d71 --- /dev/null +++ b/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCache.java @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.yangtools.yang.model.repo.util; + +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.CheckedFuture; +import com.google.common.util.concurrent.Futures; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.opendaylight.yangtools.yang.model.repo.api.MissingSchemaSourceException; +import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceException; +import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceRepresentation; +import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier; +import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; +import org.opendaylight.yangtools.yang.model.repo.spi.PotentialSchemaSource.Costs; +import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry; +import org.opendaylight.yangtools.yang.model.util.repo.FilesystemSchemaCachingProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Cache implementation that stores schemas in form of files under provided folder + */ +public final class FilesystemSchemaSourceCache extends AbstractSchemaSourceCache { + + private static final Logger LOG = LoggerFactory.getLogger(FilesystemSchemaSourceCache.class); + + // Init storage adapters + private static final Map, StorageAdapter> storageAdapters = + Collections., StorageAdapter> singletonMap( + YangTextSchemaSource.class, new YangTextSchemaStorageAdapter()); + + private final Class representation; + private final File storageDirectory; + + public FilesystemSchemaSourceCache( + final SchemaSourceRegistry consumer, final Class representation, final File storageDirectory) { + super(consumer, representation, Costs.LOCAL_IO); + this.representation = representation; + this.storageDirectory = Preconditions.checkNotNull(storageDirectory); + + checkSupportedRepresentation(representation); + + if(storageDirectory.exists() == false) { + Preconditions.checkArgument(storageDirectory.mkdirs(), "Unable to create cache directory at %s", storageDirectory); + } + Preconditions.checkArgument(storageDirectory.exists()); + Preconditions.checkArgument(storageDirectory.isDirectory()); + Preconditions.checkArgument(storageDirectory.canWrite()); + Preconditions.checkArgument(storageDirectory.canRead()); + + init(); + } + + private static void checkSupportedRepresentation(final Class representation) { + for (final Class supportedRepresentation : storageAdapters.keySet()) { + if(supportedRepresentation.isAssignableFrom(representation)) { + return; + } + } + + throw new IllegalArgumentException(String.format( + "This cache does not support representation: %s, supported representations are: %s", representation, storageAdapters.keySet())); + } + + private static final Pattern CACHED_FILE_PATTERN = + Pattern.compile( + "(?[^@]+)" + + "(@(?" + FilesystemSchemaCachingProvider.REVISION_PATTERN + "))?"); + + /** + * Restore cache state + */ + private void init() { + + final CachedModulesFileVisitor fileVisitor = new CachedModulesFileVisitor(); + try { + Files.walkFileTree(storageDirectory.toPath(), fileVisitor); + } catch (final IOException e) { + LOG.warn("Unable to restore cache from {}. Starting with empty cache", storageDirectory); + return; + } + + for (final SourceIdentifier cachedSchema : fileVisitor.getCachedSchemas()) { + register(cachedSchema); + } + } + + @Override + public synchronized CheckedFuture getSource(final SourceIdentifier sourceIdentifier) { + final File file = FilesystemSchemaCachingProvider.sourceIdToFile(toLegacy(sourceIdentifier), storageDirectory); + if(file.exists() && file.canRead()) { + LOG.trace("Source {} found in cache as {}", sourceIdentifier, file); + final SchemaSourceRepresentation restored = storageAdapters.get(representation).restore(sourceIdentifier, file); + return Futures.immediateCheckedFuture(representation.cast(restored)); + } + + LOG.debug("Source {} not found in cache as {}", sourceIdentifier, file); + return Futures.immediateFailedCheckedFuture(new MissingSchemaSourceException("Source not found")); + } + + @Override + protected synchronized void offer(final T source) { + LOG.trace("Source {} offered to cache", source.getIdentifier()); + final File file = sourceIdToFile(source); + if(file.exists()) { + LOG.debug("Source {} already in cache as {}", source.getIdentifier(), file); + return; + } + + storeSource(file, source); + register(source.getIdentifier()); + LOG.trace("Source {} stored in cache as {}", source.getIdentifier(), file); + } + + private File sourceIdToFile(final T source) { + return FilesystemSchemaCachingProvider.sourceIdToFile(toLegacy(source.getIdentifier()), storageDirectory); + } + + private void storeSource(final File file, final T schemaRepresentation) { + storageAdapters.get(representation).store(file, schemaRepresentation); + } + + private static org.opendaylight.yangtools.yang.model.util.repo.SourceIdentifier toLegacy(final SourceIdentifier identifier) { + return new org.opendaylight.yangtools.yang.model.util.repo.SourceIdentifier(identifier.getName(), Optional.fromNullable(identifier.getRevision())); + } + + private static abstract class StorageAdapter { + + private final Class supportedType; + + protected StorageAdapter(final Class supportedType) { + this.supportedType = supportedType; + } + + void store(final File file, final SchemaSourceRepresentation schemaSourceRepresentation) { + Preconditions.checkArgument(supportedType.isAssignableFrom(schemaSourceRepresentation.getClass()), + "Cannot store schema source %s, this adapter only supports %s", schemaSourceRepresentation, supportedType); + + storeAsType(file, supportedType.cast(schemaSourceRepresentation)); + + } + + protected abstract void storeAsType(final File file, final T cast); + + public Class getSupportedType() { + return supportedType; + } + + public T restore(final SourceIdentifier sourceIdentifier, final File cachedSource) { + Preconditions.checkArgument(cachedSource.isFile()); + Preconditions.checkArgument(cachedSource.exists()); + Preconditions.checkArgument(cachedSource.canRead()); + return restoreAsType(sourceIdentifier, cachedSource); + } + + protected abstract T restoreAsType(final SourceIdentifier sourceIdentifier, final File cachedSource); + } + + private static final class YangTextSchemaStorageAdapter extends StorageAdapter { + + protected YangTextSchemaStorageAdapter() { + super(YangTextSchemaSource.class); + } + + @Override + protected void storeAsType(final File file, final YangTextSchemaSource cast) { + try { + Files.copy(cast.openStream(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (final IOException e) { + throw new IllegalStateException("Cannot store schema source " + cast.getIdentifier() + " to " + file, e); + } + } + + @Override + public YangTextSchemaSource restoreAsType(final SourceIdentifier sourceIdentifier, final File cachedSource) { + return new YangTextSchemaSource(sourceIdentifier) { + + @Override + protected Objects.ToStringHelper addToStringAttributes(final Objects.ToStringHelper toStringHelper) { + return toStringHelper; + } + + @Override + public InputStream openStream() throws IOException { + return new FileInputStream(cachedSource); + } + }; + } + } + + private static final class CachedModulesFileVisitor extends SimpleFileVisitor { + private final List cachedSchemas = Lists.newArrayList(); + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + final FileVisitResult fileVisitResult = super.visitFile(file, attrs); + String fileName = file.toFile().getName(); + fileName = com.google.common.io.Files.getNameWithoutExtension(fileName); + + final Optional si = getSourceIdentifier(fileName); + if(si.isPresent()) { + LOG.trace("Restoring cached file {} as {}", file, si.get()); + cachedSchemas.add(si.get()); + } else { + LOG.debug("Skipping cached file {}, cannot restore source identifier from filename: {}, does not match {}", file, fileName, CACHED_FILE_PATTERN); + } + return fileVisitResult; + } + + private Optional getSourceIdentifier(final String fileName) { + final Matcher matcher = CACHED_FILE_PATTERN.matcher(fileName); + if(matcher.matches()) { + final String moduleName = matcher.group("moduleName"); + final String revision = matcher.group("revision"); + return Optional.of(new SourceIdentifier(moduleName, Optional.fromNullable(revision))); + } + return Optional.absent(); + } + + @Override + public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException { + LOG.warn("Unable to restore cached file {}. Ignoring", file, exc); + return FileVisitResult.CONTINUE; + } + + public List getCachedSchemas() { + return cachedSchemas; + } + } +} diff --git a/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/util/repo/FilesystemSchemaCachingProvider.java b/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/util/repo/FilesystemSchemaCachingProvider.java index 42d6df985f..4e80033dc3 100644 --- a/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/util/repo/FilesystemSchemaCachingProvider.java +++ b/yang/yang-model-util/src/main/java/org/opendaylight/yangtools/yang/model/util/repo/FilesystemSchemaCachingProvider.java @@ -45,7 +45,7 @@ import com.google.common.base.Preconditions; */ public final class FilesystemSchemaCachingProvider extends AbstractCachingSchemaSourceProvider { private static final Logger LOG = LoggerFactory.getLogger(FilesystemSchemaCachingProvider.class); - private static final Pattern REVISION_PATTERN = Pattern.compile("\\d\\d\\d\\d-\\d\\d-\\d\\d"); + public static final Pattern REVISION_PATTERN = Pattern.compile("\\d\\d\\d\\d-\\d\\d-\\d\\d"); private final File storageDirectory; private final SchemaSourceTransformation transformationFunction; @@ -154,17 +154,21 @@ public final class FilesystemSchemaCachingProvider extends AbstractCachingSch } private File toFile(final SourceIdentifier identifier) { + return sourceIdToFile(identifier, storageDirectory); + } + + public static File sourceIdToFile(final SourceIdentifier identifier, final File storageDirectory) { File file = null; String rev = identifier.getRevision(); if (rev == null || rev.isEmpty()) { - file = findFileWithNewestRev(identifier); + file = findFileWithNewestRev(identifier, storageDirectory); } else { file = new File(storageDirectory, identifier.toYangFilename()); } return file; } - private File findFileWithNewestRev(final SourceIdentifier identifier) { + private static File findFileWithNewestRev(final SourceIdentifier identifier, final File storageDirectory) { File[] files = storageDirectory.listFiles(new FilenameFilter() { final Pattern p = Pattern.compile(Pattern.quote(identifier.getName()) + "(\\.yang|@\\d\\d\\d\\d-\\d\\d-\\d\\d.yang)"); diff --git a/yang/yang-model-util/src/test/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCacheTest.java b/yang/yang-model-util/src/test/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCacheTest.java new file mode 100644 index 0000000000..1cb9ed7746 --- /dev/null +++ b/yang/yang-model-util/src/test/java/org/opendaylight/yangtools/yang/model/repo/util/FilesystemSchemaSourceCacheTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + +package org.opendaylight.yangtools.yang.model.repo.util; + +import static org.hamcrest.CoreMatchers.both; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.either; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.common.base.Charsets; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.google.common.collect.Collections2; +import com.google.common.io.Files; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier; +import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; +import org.opendaylight.yangtools.yang.model.repo.spi.PotentialSchemaSource; +import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceProvider; +import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistration; +import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry; + +public class FilesystemSchemaSourceCacheTest { + + @Mock + private SchemaSourceRegistry registry; + @Mock + private SchemaSourceRegistration registration; + private File storageDir; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + storageDir = Files.createTempDir(); + doReturn(registration).when(registry).registerSchemaSource(any(SchemaSourceProvider.class), any(PotentialSchemaSource.class)); + } + + @Test + public void testCacheAndRestore() throws Exception { + final FilesystemSchemaSourceCache cache + = new FilesystemSchemaSourceCache<>(registry, YangTextSchemaSource.class, storageDir); + + final String content = "content1"; + final YangTextSchemaSource source = new TestingYangSource("test", "2012-12-12", content); + cache.offer(source); + + final String content2 = "content2"; + final YangTextSchemaSource source2 = new TestingYangSource("test2", null, content); + cache.offer(source2); + + final List storedFiles = getFilesFromCache(); + assertEquals(2, storedFiles.size()); + final Collection fileNames = filesToFilenamesWithoutRevision(storedFiles); + + assertThat(fileNames, both(hasItem("test2@0000-00-00")).and(hasItem("test@2012-12-12"))); + + assertThat(Files.toString(storedFiles.get(0), Charsets.UTF_8), either(containsString(content)).or(containsString(content2))); + assertThat(Files.toString(storedFiles.get(1), Charsets.UTF_8), either(containsString(content)).or(containsString(content2))); + + verify(registry, times(2)).registerSchemaSource(any(SchemaSourceProvider.class), any(PotentialSchemaSource.class)); + + // Create new cache from stored sources + new FilesystemSchemaSourceCache<>(registry, YangTextSchemaSource.class, storageDir); + + verify(registry, times(4)).registerSchemaSource(any(SchemaSourceProvider.class), any(PotentialSchemaSource.class)); + + final List storedFilesAfterNewCache = getFilesFromCache(); + assertEquals(2, storedFilesAfterNewCache.size()); + } + + private Collection filesToFilenamesWithoutRevision(final List storedFiles) { + return Collections2.transform(storedFiles, new Function() { + @Override + public String apply(final File input) { + return Files.getNameWithoutExtension(input.getName()); + } + }); + } + + @Test + public void testCacheDuplicate() throws Exception { + final FilesystemSchemaSourceCache cache + = new FilesystemSchemaSourceCache<>(registry, YangTextSchemaSource.class, storageDir); + + final String content = "content1"; + final YangTextSchemaSource source = new TestingYangSource("test", null, content); + // Double offer + cache.offer(source); + cache.offer(source); + + final List storedFiles = getFilesFromCache(); + assertEquals(1, storedFiles.size()); + verify(registry).registerSchemaSource(any(SchemaSourceProvider.class), any(PotentialSchemaSource.class)); + } + + @Test + public void testCacheMultipleRevisions() throws Exception { + final FilesystemSchemaSourceCache cache + = new FilesystemSchemaSourceCache<>(registry, YangTextSchemaSource.class, storageDir); + + final String content = "content1"; + final YangTextSchemaSource source = new TestingYangSource("test", null, content); + final YangTextSchemaSource source2 = new TestingYangSource("test", "2012-12-12", content); + final YangTextSchemaSource source3 = new TestingYangSource("test", "2013-12-12", content); + // Double offer + cache.offer(source); + cache.offer(source2); + cache.offer(source3); + + final List storedFiles = getFilesFromCache(); + assertEquals(3, storedFiles.size()); + + assertThat(filesToFilenamesWithoutRevision(storedFiles), both(hasItem("test@0000-00-00")).and(hasItem("test@2012-12-12")).and(hasItem("test@2013-12-12"))); + + verify(registry, times(3)).registerSchemaSource(any(SchemaSourceProvider.class), any(PotentialSchemaSource.class)); + } + + private List getFilesFromCache() { + return Arrays.asList(storageDir.listFiles()); + } + + private class TestingYangSource extends YangTextSchemaSource { + + private final String content; + + protected TestingYangSource(final String name, final String revision, final String content) { + super(new SourceIdentifier(name, Optional.fromNullable(revision))); + this.content = content; + } + + @Override + protected Objects.ToStringHelper addToStringAttributes(final Objects.ToStringHelper toStringHelper) { + return toStringHelper; + } + + @Override + public InputStream openStream() throws IOException { + return new ByteArrayInputStream(content.getBytes(Charsets.UTF_8)); + } + } +} diff --git a/yang/yang-parser-impl/src/test/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaRepositoryTest.java b/yang/yang-parser-impl/src/test/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaRepositoryTest.java index ac7320c02c..9a8790a428 100644 --- a/yang/yang-parser-impl/src/test/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaRepositoryTest.java +++ b/yang/yang-parser-impl/src/test/java/org/opendaylight/yangtools/yang/parser/repo/SharedSchemaRepositoryTest.java @@ -14,24 +14,47 @@ import static junit.framework.Assert.assertNotNull; import static junit.framework.Assert.assertSame; import static junit.framework.Assert.assertTrue; import static junit.framework.Assert.fail; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.matchers.JUnitMatchers.both; +import static org.junit.matchers.JUnitMatchers.hasItem; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import org.junit.Ignore; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import com.google.common.io.InputSupplier; +import com.google.common.util.concurrent.CheckedFuture; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutionException; +import org.apache.commons.io.IOUtils; import org.junit.Test; import org.opendaylight.yangtools.yang.model.api.SchemaContext; +import org.opendaylight.yangtools.yang.model.repo.api.MissingSchemaSourceException; import org.opendaylight.yangtools.yang.model.repo.api.SchemaContextFactory; import org.opendaylight.yangtools.yang.model.repo.api.SchemaResolutionException; import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceException; import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceFilter; +import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceRepresentation; import org.opendaylight.yangtools.yang.model.repo.api.SourceIdentifier; +import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; +import org.opendaylight.yangtools.yang.model.repo.spi.PotentialSchemaSource; +import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceListener; +import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceProvider; +import org.opendaylight.yangtools.yang.model.repo.util.FilesystemSchemaSourceCache; import org.opendaylight.yangtools.yang.parser.util.ASTSchemaSource; import org.opendaylight.yangtools.yang.parser.util.TextToASTTransformer; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.CheckedFuture; - public class SharedSchemaRepositoryTest { @Test @@ -127,8 +150,6 @@ public class SharedSchemaRepositoryTest { fail("Schema context creation should have failed"); } - // TODO - @Ignore("Costs are not considered when getting sources") @Test public void testDifferentCosts() throws Exception { final SharedSchemaRepository sharedSchemaRepository = new SharedSchemaRepository("netconf-mounts"); @@ -153,6 +174,125 @@ public class SharedSchemaRepositoryTest { verify(immediateInetTypesYang).getSource(id); } + @Test + public void testWithCacheStartup() throws Exception { + final SharedSchemaRepository sharedSchemaRepository = new SharedSchemaRepository("netconf-mounts"); + + class CountingSchemaListener implements SchemaSourceListener { + List> registeredSources = Lists.newArrayList(); + + @Override + public void schemaSourceEncountered(final SchemaSourceRepresentation source) { + } + + @Override + public void schemaSourceRegistered(final Iterable> sources) { + for (final PotentialSchemaSource source : sources) { + registeredSources.add(source); + } + } + + @Override + public void schemaSourceUnregistered(final PotentialSchemaSource source) { + } + } + + final File storageDir = Files.createTempDir(); + + final CountingSchemaListener listener = new CountingSchemaListener(); + sharedSchemaRepository.registerSchemaSourceListener(listener); + + final File test = new File(storageDir, "test.yang"); + Files.copy(new StringSupplier("content-test"), test); + + final File test2 = new File(storageDir, "test@2012-12-12.yang"); + Files.copy(new StringSupplier("content-test-2012"), test2); + + final File test3 = new File(storageDir, "test@2013-12-12.yang"); + Files.copy(new StringSupplier("content-test-2013"), test3); + + final File test4 = new File(storageDir, "module@2010-12-12.yang"); + Files.copy(new StringSupplier("content-module-2010"), test4); + + + final FilesystemSchemaSourceCache cache = new FilesystemSchemaSourceCache<>(sharedSchemaRepository, YangTextSchemaSource.class, storageDir); + sharedSchemaRepository.registerSchemaSourceListener(cache); + + assertEquals(4, listener.registeredSources.size()); + + final Function, SourceIdentifier> potSourceToSID = new Function, SourceIdentifier>() { + @Override + public SourceIdentifier apply(final PotentialSchemaSource input) { + return input.getSourceIdentifier(); + } + }; + assertThat(Collections2.transform(listener.registeredSources, potSourceToSID), + both(hasItem(new SourceIdentifier("test", Optional.absent()))) + .and(hasItem(new SourceIdentifier("test", Optional.of("2012-12-12")))) + .and(hasItem(new SourceIdentifier("test", Optional.of("2013-12-12")))) + .and(hasItem(new SourceIdentifier("module", Optional.of("2010-12-12")))) + ); + } + + @Test + public void testWithCacheRunning() throws Exception { + final SharedSchemaRepository sharedSchemaRepository = new SharedSchemaRepository("netconf-mounts"); + + final File storageDir = Files.createTempDir(); + + final FilesystemSchemaSourceCache cache = new FilesystemSchemaSourceCache<>(sharedSchemaRepository, YangTextSchemaSource.class, storageDir); + sharedSchemaRepository.registerSchemaSourceListener(cache); + + final SourceIdentifier runningId = new SourceIdentifier("running", Optional.of("2012-12-12")); + + sharedSchemaRepository.registerSchemaSource(new SchemaSourceProvider() { + @Override + public CheckedFuture getSource(final SourceIdentifier sourceIdentifier) { + return Futures.immediateCheckedFuture(new YangTextSchemaSource(runningId) { + @Override + protected Objects.ToStringHelper addToStringAttributes(final Objects.ToStringHelper toStringHelper) { + return toStringHelper; + } + + @Override + public InputStream openStream() throws IOException { + return IOUtils.toInputStream("running"); + } + }); + } + }, PotentialSchemaSource.create(runningId, YangTextSchemaSource.class, PotentialSchemaSource.Costs.REMOTE_IO.getValue())); + + final TextToASTTransformer transformer = TextToASTTransformer.create(sharedSchemaRepository, sharedSchemaRepository); + sharedSchemaRepository.registerSchemaSourceListener(transformer); + + // Request schema to make repository notify the cache + final CheckedFuture schemaFuture = sharedSchemaRepository.createSchemaContextFactory(SchemaSourceFilter.ALWAYS_ACCEPT).createSchemaContext(Lists.newArrayList(runningId)); + Futures.addCallback(schemaFuture, new FutureCallback() { + @Override + public void onSuccess(final SchemaContext result) { + fail("Creation of schema context should fail from non-regular sources"); + } + + @Override + public void onFailure(final Throwable t) { + // Creation of schema context fails, since we do not provide regular sources, but we just want to check cache + final List cachedSchemas = Arrays.asList(storageDir.listFiles()); + assertEquals(1, cachedSchemas.size()); + assertEquals(Files.getNameWithoutExtension(cachedSchemas.get(0).getName()), "running@2012-12-12"); + } + }); + + try { + schemaFuture.get(); + } catch (final ExecutionException e) { + assertNotNull(e.getCause()); + assertEquals(MissingSchemaSourceException.class, e.getCause().getClass()); + return; + } + + fail("Creation of schema context should fail from non-regular sources"); + } + private void assertSchemaContext(final SchemaContext schemaContext, final int moduleSize) { assertNotNull(schemaContext); assertEquals(moduleSize, schemaContext.getModules().size()); @@ -170,4 +310,16 @@ public class SharedSchemaRepositoryTest { return SettableSchemaProvider.createImmediate(aSTSchemaSource.get(), ASTSchemaSource.class); } + private class StringSupplier implements InputSupplier { + private final String s; + + public StringSupplier(final String s) { + this.s = s; + } + + @Override + public InputStream getInput() throws IOException { + return IOUtils.toInputStream(s); + } + } } -- 2.36.6