Add AST yang model caching to netconf connector's schema repository

text -> AST transformation of YANG takes majority of the time when tryinh to parse
SchemaContext. Caching is important when mounting a device where not all yang models
are valid or have their dependencies. Netconf connector tires to parse and eliminate
YANG models in a loop until a valid subset is found. Until now, the text -> AST has
been performed in each iteration, really slowing donw the mount process.

Also fix logs in NetconfDevice

Change-Id: I0587299eefdc7a813afd9967363200243b815551
Signed-off-by: Maros Marsalek <mmarsalek@frinx.io>

diff --git a/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java b/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java
index 21ff0010526bb8f6f9ec8fa49999490f8db49478..eff84bba1fd2e1b9f9be22a8384dca8ed739b296 100644 (file)
--- a/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java
+++ b/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java
@@ -36,7 +36,9 @@ import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceFilter;
 import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource;
 import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry;
 import org.opendaylight.yangtools.yang.model.repo.util.FilesystemSchemaSourceCache;
+import org.opendaylight.yangtools.yang.model.repo.util.InMemorySchemaSourceCache;
 import org.opendaylight.yangtools.yang.parser.repo.SharedSchemaRepository;
+import org.opendaylight.yangtools.yang.parser.rfc7950.repo.ASTSchemaSource;
 import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToASTTransformer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -77,6 +79,9 @@ public final class NetconfTopologyUtils {
             new FilesystemSchemaSourceCache<>(DEFAULT_SCHEMA_REPOSITORY, YangTextSchemaSource.class,
                     new File(QUALIFIED_DEFAULT_CACHE_DIRECTORY));
 
+    public static final InMemorySchemaSourceCache<ASTSchemaSource> DEFAULT_AST_CACHE =
+            InMemorySchemaSourceCache.createSoftCache(DEFAULT_SCHEMA_REPOSITORY, ASTSchemaSource.class);
+
     // The default factory for creating <code>SchemaContext</code> instances.
     public static final SchemaContextFactory DEFAULT_SCHEMA_CONTEXT_FACTORY =
             DEFAULT_SCHEMA_REPOSITORY.createSchemaContextFactory(SchemaSourceFilter.ALWAYS_ACCEPT);
@@ -98,6 +103,7 @@ public final class NetconfTopologyUtils {
                 new NetconfDevice.SchemaResourcesDTO(DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_REPOSITORY,
                         DEFAULT_SCHEMA_CONTEXT_FACTORY, new NetconfStateSchemasResolverImpl()));
         DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_CACHE);
+        DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_AST_CACHE);
         DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(
                 TextToASTTransformer.create(DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_REPOSITORY));
     }
@@ -164,6 +170,8 @@ public final class NetconfTopologyUtils {
         final FilesystemSchemaSourceCache<YangTextSchemaSource> deviceCache =
                 createDeviceFilesystemCache(moduleSchemaCacheDirectory, repository);
         repository.registerSchemaSourceListener(deviceCache);
+        repository.registerSchemaSourceListener(InMemorySchemaSourceCache.createSoftCache(repository,
+                ASTSchemaSource.class));
         return new NetconfDevice.SchemaResourcesDTO(repository, repository, schemaContextFactory,
                 new NetconfStateSchemasResolverImpl());
     }

diff --git a/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java b/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java
index e3a84195332f6713962567f1307e7cc4af1a41d4..5f7f3594dedc5df2ff7cf6bfdf246779b9da7676 100644 (file)
--- a/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java
+++ b/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java
@@ -95,7 +95,9 @@ import org.opendaylight.yangtools.yang.model.repo.spi.PotentialSchemaSource;
 import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistration;
 import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry;
 import org.opendaylight.yangtools.yang.model.repo.util.FilesystemSchemaSourceCache;
+import org.opendaylight.yangtools.yang.model.repo.util.InMemorySchemaSourceCache;
 import org.opendaylight.yangtools.yang.parser.repo.SharedSchemaRepository;
+import org.opendaylight.yangtools.yang.parser.rfc7950.repo.ASTSchemaSource;
 import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToASTTransformer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -148,6 +150,9 @@ public abstract class AbstractNetconfTopology implements NetconfTopology {
             new FilesystemSchemaSourceCache<>(DEFAULT_SCHEMA_REPOSITORY, YangTextSchemaSource.class,
                     new File(QUALIFIED_DEFAULT_CACHE_DIRECTORY));
 
+    public static final InMemorySchemaSourceCache<ASTSchemaSource> DEFAULT_AST_CACHE =
+            InMemorySchemaSourceCache.createSoftCache(DEFAULT_SCHEMA_REPOSITORY, ASTSchemaSource.class);
+
     /**
      * The default factory for creating <code>SchemaContext</code> instances.
      */
@@ -172,6 +177,7 @@ public abstract class AbstractNetconfTopology implements NetconfTopology {
                         DEFAULT_SCHEMA_CONTEXT_FACTORY,
                         new NetconfStateSchemasResolverImpl()));
         DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_CACHE);
+        DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_AST_CACHE);
         DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(
                 TextToASTTransformer.create(DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_REPOSITORY));
     }
@@ -410,6 +416,8 @@ public abstract class AbstractNetconfTopology implements NetconfTopology {
         final FilesystemSchemaSourceCache<YangTextSchemaSource> deviceCache =
                 createDeviceFilesystemCache(moduleSchemaCacheDirectory);
         repository.registerSchemaSourceListener(deviceCache);
+        repository.registerSchemaSourceListener(
+            InMemorySchemaSourceCache.createSoftCache(repository, ASTSchemaSource.class));
         return new NetconfDevice.SchemaResourcesDTO(repository, repository, contextFactory,
                 new NetconfStateSchemasResolverImpl());
     }

diff --git a/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java b/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java
index 0abb0ea622f1e993743edbe971166a69bf2a2ff0..c9ccf93fcbcf54385d1f99179218da152c55636e 100644 (file)
--- a/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java
+++ b/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java
@@ -537,12 +537,12 @@ public class NetconfDevice
                 final Collection<SourceIdentifier> requiredSources, final MissingSchemaSourceException exception) {
             // In case source missing, try without it
             final SourceIdentifier missingSource = exception.getSourceId();
-            LOG.warn("{}: Unable to build schema context, missing source {}, will reattempt without it", id,
-                    missingSource);
+            LOG.warn("{}: Unable to build schema context, missing source {}, will reattempt without it",
+                id, missingSource);
             LOG.debug("{}: Unable to build schema context, missing source {}, will reattempt without it",
-                    exception);
+                id, missingSource, exception);
             final Collection<QName> qNameOfMissingSource =
-                    getQNameFromSourceIdentifiers(Sets.newHashSet(missingSource));
+                getQNameFromSourceIdentifiers(Sets.newHashSet(missingSource));
             if (!qNameOfMissingSource.isEmpty()) {
                 capabilities.addUnresolvedCapabilities(
                         qNameOfMissingSource, UnavailableCapability.FailureReason.MissingSource);
@@ -559,21 +559,22 @@ public class NetconfDevice
                 // flawed model - exclude it
                 final SourceIdentifier failedSourceId = resolutionException.getFailedSource();
                 LOG.warn("{}: Unable to build schema context, failed to resolve source {}, will reattempt without it",
-                        id, failedSourceId);
+                    id, failedSourceId);
                 LOG.warn("{}: Unable to build schema context, failed to resolve source {}, will reattempt without it",
-                        id, resolutionException);
-                capabilities.addUnresolvedCapabilities(getQNameFromSourceIdentifiers(
-                        Collections.singleton(failedSourceId)), UnavailableCapability.FailureReason.UnableToResolve);
+                    id, failedSourceId, resolutionException);
+                capabilities.addUnresolvedCapabilities(
+                        getQNameFromSourceIdentifiers(Collections.singleton(failedSourceId)),
+                        UnavailableCapability.FailureReason.UnableToResolve);
                 return stripUnavailableSource(requiredSources, resolutionException.getFailedSource());
             }
             // unsatisfied imports
             final Set<SourceIdentifier> unresolvedSources = resolutionException.getUnsatisfiedImports().keySet();
-            capabilities.addUnresolvedCapabilities(
-                getQNameFromSourceIdentifiers(unresolvedSources), UnavailableCapability.FailureReason.UnableToResolve);
+            capabilities.addUnresolvedCapabilities(getQNameFromSourceIdentifiers(unresolvedSources),
+                UnavailableCapability.FailureReason.UnableToResolve);
             LOG.warn("{}: Unable to build schema context, unsatisfied imports {}, will reattempt with resolved only",
-                    id, resolutionException.getUnsatisfiedImports());
+                id, resolutionException.getUnsatisfiedImports());
             LOG.debug("{}: Unable to build schema context, unsatisfied imports {}, will reattempt with resolved only",
-                    resolutionException);
+                id, resolutionException.getUnsatisfiedImports(), resolutionException);
             return resolutionException.getResolvedSources();
         }