From db1f73a19df067d73ad72551e784a11ab837e34e Mon Sep 17 00:00:00 2001 From: Maros Marsalek Date: Thu, 10 May 2018 16:27:49 +0200 Subject: [PATCH] Add AST yang model caching to netconf connector's schema repository text -> AST transformation of YANG takes majority of the time when tryinh to parse SchemaContext. Caching is important when mounting a device where not all yang models are valid or have their dependencies. Netconf connector tires to parse and eliminate YANG models in a loop until a valid subset is found. Until now, the text -> AST has been performed in each iteration, really slowing donw the mount process. Also fix logs in NetconfDevice Change-Id: I0587299eefdc7a813afd9967363200243b815551 Signed-off-by: Maros Marsalek --- .../impl/utils/NetconfTopologyUtils.java | 8 ++++++ .../topology/AbstractNetconfTopology.java | 8 ++++++ .../sal/connect/netconf/NetconfDevice.java | 25 ++++++++++--------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java b/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java index 21ff001052..eff84bba1f 100644 --- a/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java +++ b/netconf/netconf-topology-singleton/src/main/java/org/opendaylight/netconf/topology/singleton/impl/utils/NetconfTopologyUtils.java @@ -36,7 +36,9 @@ import org.opendaylight.yangtools.yang.model.repo.api.SchemaSourceFilter; import org.opendaylight.yangtools.yang.model.repo.api.YangTextSchemaSource; import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry; import org.opendaylight.yangtools.yang.model.repo.util.FilesystemSchemaSourceCache; +import org.opendaylight.yangtools.yang.model.repo.util.InMemorySchemaSourceCache; import org.opendaylight.yangtools.yang.parser.repo.SharedSchemaRepository; +import org.opendaylight.yangtools.yang.parser.rfc7950.repo.ASTSchemaSource; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToASTTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,6 +79,9 @@ public final class NetconfTopologyUtils { new FilesystemSchemaSourceCache<>(DEFAULT_SCHEMA_REPOSITORY, YangTextSchemaSource.class, new File(QUALIFIED_DEFAULT_CACHE_DIRECTORY)); + public static final InMemorySchemaSourceCache DEFAULT_AST_CACHE = + InMemorySchemaSourceCache.createSoftCache(DEFAULT_SCHEMA_REPOSITORY, ASTSchemaSource.class); + // The default factory for creating SchemaContext instances. public static final SchemaContextFactory DEFAULT_SCHEMA_CONTEXT_FACTORY = DEFAULT_SCHEMA_REPOSITORY.createSchemaContextFactory(SchemaSourceFilter.ALWAYS_ACCEPT); @@ -98,6 +103,7 @@ public final class NetconfTopologyUtils { new NetconfDevice.SchemaResourcesDTO(DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_CONTEXT_FACTORY, new NetconfStateSchemasResolverImpl())); DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_CACHE); + DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_AST_CACHE); DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener( TextToASTTransformer.create(DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_REPOSITORY)); } @@ -164,6 +170,8 @@ public final class NetconfTopologyUtils { final FilesystemSchemaSourceCache deviceCache = createDeviceFilesystemCache(moduleSchemaCacheDirectory, repository); repository.registerSchemaSourceListener(deviceCache); + repository.registerSchemaSourceListener(InMemorySchemaSourceCache.createSoftCache(repository, + ASTSchemaSource.class)); return new NetconfDevice.SchemaResourcesDTO(repository, repository, schemaContextFactory, new NetconfStateSchemasResolverImpl()); } diff --git a/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java b/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java index e3a8419533..5f7f3594de 100644 --- a/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java +++ b/netconf/netconf-topology/src/main/java/org/opendaylight/netconf/topology/AbstractNetconfTopology.java @@ -95,7 +95,9 @@ import org.opendaylight.yangtools.yang.model.repo.spi.PotentialSchemaSource; import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistration; import org.opendaylight.yangtools.yang.model.repo.spi.SchemaSourceRegistry; import org.opendaylight.yangtools.yang.model.repo.util.FilesystemSchemaSourceCache; +import org.opendaylight.yangtools.yang.model.repo.util.InMemorySchemaSourceCache; import org.opendaylight.yangtools.yang.parser.repo.SharedSchemaRepository; +import org.opendaylight.yangtools.yang.parser.rfc7950.repo.ASTSchemaSource; import org.opendaylight.yangtools.yang.parser.rfc7950.repo.TextToASTTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,6 +150,9 @@ public abstract class AbstractNetconfTopology implements NetconfTopology { new FilesystemSchemaSourceCache<>(DEFAULT_SCHEMA_REPOSITORY, YangTextSchemaSource.class, new File(QUALIFIED_DEFAULT_CACHE_DIRECTORY)); + public static final InMemorySchemaSourceCache DEFAULT_AST_CACHE = + InMemorySchemaSourceCache.createSoftCache(DEFAULT_SCHEMA_REPOSITORY, ASTSchemaSource.class); + /** * The default factory for creating SchemaContext instances. */ @@ -172,6 +177,7 @@ public abstract class AbstractNetconfTopology implements NetconfTopology { DEFAULT_SCHEMA_CONTEXT_FACTORY, new NetconfStateSchemasResolverImpl())); DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_CACHE); + DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener(DEFAULT_AST_CACHE); DEFAULT_SCHEMA_REPOSITORY.registerSchemaSourceListener( TextToASTTransformer.create(DEFAULT_SCHEMA_REPOSITORY, DEFAULT_SCHEMA_REPOSITORY)); } @@ -410,6 +416,8 @@ public abstract class AbstractNetconfTopology implements NetconfTopology { final FilesystemSchemaSourceCache deviceCache = createDeviceFilesystemCache(moduleSchemaCacheDirectory); repository.registerSchemaSourceListener(deviceCache); + repository.registerSchemaSourceListener( + InMemorySchemaSourceCache.createSoftCache(repository, ASTSchemaSource.class)); return new NetconfDevice.SchemaResourcesDTO(repository, repository, contextFactory, new NetconfStateSchemasResolverImpl()); } diff --git a/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java b/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java index 0abb0ea622..c9ccf93fcb 100644 --- a/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java +++ b/netconf/sal-netconf-connector/src/main/java/org/opendaylight/netconf/sal/connect/netconf/NetconfDevice.java @@ -537,12 +537,12 @@ public class NetconfDevice final Collection requiredSources, final MissingSchemaSourceException exception) { // In case source missing, try without it final SourceIdentifier missingSource = exception.getSourceId(); - LOG.warn("{}: Unable to build schema context, missing source {}, will reattempt without it", id, - missingSource); + LOG.warn("{}: Unable to build schema context, missing source {}, will reattempt without it", + id, missingSource); LOG.debug("{}: Unable to build schema context, missing source {}, will reattempt without it", - exception); + id, missingSource, exception); final Collection qNameOfMissingSource = - getQNameFromSourceIdentifiers(Sets.newHashSet(missingSource)); + getQNameFromSourceIdentifiers(Sets.newHashSet(missingSource)); if (!qNameOfMissingSource.isEmpty()) { capabilities.addUnresolvedCapabilities( qNameOfMissingSource, UnavailableCapability.FailureReason.MissingSource); @@ -559,21 +559,22 @@ public class NetconfDevice // flawed model - exclude it final SourceIdentifier failedSourceId = resolutionException.getFailedSource(); LOG.warn("{}: Unable to build schema context, failed to resolve source {}, will reattempt without it", - id, failedSourceId); + id, failedSourceId); LOG.warn("{}: Unable to build schema context, failed to resolve source {}, will reattempt without it", - id, resolutionException); - capabilities.addUnresolvedCapabilities(getQNameFromSourceIdentifiers( - Collections.singleton(failedSourceId)), UnavailableCapability.FailureReason.UnableToResolve); + id, failedSourceId, resolutionException); + capabilities.addUnresolvedCapabilities( + getQNameFromSourceIdentifiers(Collections.singleton(failedSourceId)), + UnavailableCapability.FailureReason.UnableToResolve); return stripUnavailableSource(requiredSources, resolutionException.getFailedSource()); } // unsatisfied imports final Set unresolvedSources = resolutionException.getUnsatisfiedImports().keySet(); - capabilities.addUnresolvedCapabilities( - getQNameFromSourceIdentifiers(unresolvedSources), UnavailableCapability.FailureReason.UnableToResolve); + capabilities.addUnresolvedCapabilities(getQNameFromSourceIdentifiers(unresolvedSources), + UnavailableCapability.FailureReason.UnableToResolve); LOG.warn("{}: Unable to build schema context, unsatisfied imports {}, will reattempt with resolved only", - id, resolutionException.getUnsatisfiedImports()); + id, resolutionException.getUnsatisfiedImports()); LOG.debug("{}: Unable to build schema context, unsatisfied imports {}, will reattempt with resolved only", - resolutionException); + id, resolutionException.getUnsatisfiedImports(), resolutionException); return resolutionException.getResolvedSources(); } -- 2.36.6