From: Tony Tkacik Date: Wed, 22 Jan 2014 14:54:00 +0000 (+0100) Subject: Introduced advanced strategy for parsing of YANG schemas X-Git-Tag: jenkins-controller-bulk-release-prepare-only-2-1~12 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=3c6f666df68dfa61608dba4566afefb168fc7e7a Introduced advanced strategy for parsing of YANG schemas - This strategy increases convergence time, because it does not parse files with unsatisfied dependencies. - New strategy for schema parsing in runtime uses two separate parser Minimalistic - is used when YANG source is discovered and scans only for module name, revision and dependency information Full - parses set of YANG sources and creates model representation. When bundle is added / removed - tracker gets yang sources, registers it to SchemaResolver, which immediatelly uses Minimalistic parser to obtain base information. Then context of all known sources and their dependencies is sorted and filtered only to use sources which has their dependencies resolved. Full parser is then invoked only for models, which have dependencies resolved. Change-Id: Ic62fbd86929866a21f4dbd3c62e73a02d423d7bf Signed-off-by: Tony Tkacik --- diff --git a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/config/yang/md/sal/dom/impl/SchemaServiceImplSingletonModule.java b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/config/yang/md/sal/dom/impl/SchemaServiceImplSingletonModule.java index 6b597dea9e..d61d709600 100644 --- a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/config/yang/md/sal/dom/impl/SchemaServiceImplSingletonModule.java +++ b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/config/yang/md/sal/dom/impl/SchemaServiceImplSingletonModule.java @@ -51,7 +51,6 @@ public final class SchemaServiceImplSingletonModule extends org.opendaylight.con public java.lang.AutoCloseable createInstance() { SchemaServiceImpl newInstance = new SchemaServiceImpl(); newInstance.setContext(getBundleContext()); - newInstance.setParser(new YangParserImpl()); newInstance.start(); return newInstance; } diff --git a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/BrokerConfigActivator.xtend b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/BrokerConfigActivator.xtend index 3baae04019..b5737a5454 100644 --- a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/BrokerConfigActivator.xtend +++ b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/BrokerConfigActivator.xtend @@ -1,73 +1,75 @@ +/* + * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ package org.opendaylight.controller.sal.dom.broker -import org.osgi.framework.ServiceRegistration -import org.opendaylight.controller.sal.core.api.model.SchemaService +import java.util.Hashtable import org.opendaylight.controller.sal.core.api.data.DataBrokerService import org.opendaylight.controller.sal.core.api.data.DataProviderService -import org.opendaylight.controller.sal.dom.broker.impl.HashMapDataStore +import org.opendaylight.controller.sal.core.api.data.DataStore +import org.opendaylight.controller.sal.core.api.model.SchemaService +import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener import org.opendaylight.controller.sal.core.api.mount.MountProvisionService import org.opendaylight.controller.sal.core.api.mount.MountService -import org.osgi.framework.BundleContext -import java.util.Hashtable -import org.opendaylight.yangtools.yang.parser.impl.YangParserImpl -import org.opendaylight.yangtools.yang.data.api.InstanceIdentifier -import org.opendaylight.controller.sal.core.api.data.DataStore import org.opendaylight.controller.sal.dom.broker.impl.SchemaAwareDataStoreAdapter -import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener import org.opendaylight.controller.sal.dom.broker.impl.SchemaAwareRpcBroker +import org.opendaylight.yangtools.yang.data.api.InstanceIdentifier +import org.osgi.framework.BundleContext +import org.osgi.framework.ServiceRegistration +import org.opendaylight.controller.sal.dom.broker.impl.SchemaContextProviders class BrokerConfigActivator implements AutoCloseable { - - + private static val ROOT = InstanceIdentifier.builder().toInstance(); @Property private var DataBrokerImpl dataService; - + private var ServiceRegistration schemaReg; private var ServiceRegistration dataReg; private var ServiceRegistration dataProviderReg; private var ServiceRegistration mountReg; private var ServiceRegistration mountProviderReg; - private var SchemaServiceImpl schemaService; + private var SchemaService schemaService; private var MountPointManagerImpl mountService; - + SchemaAwareDataStoreAdapter wrappedStore - public def void start(BrokerImpl broker,DataStore store,BundleContext context) { + public def void start(BrokerImpl broker, DataStore store, BundleContext context) { val emptyProperties = new Hashtable(); broker.setBundleContext(context); - - - schemaService = new SchemaServiceImpl(); - schemaService.setContext(context); - schemaService.setParser(new YangParserImpl()); - schemaService.start(); + + val serviceRef = context.getServiceReference(SchemaService); + schemaService = context.getService(serviceRef); schemaReg = context.registerService(SchemaService, schemaService, emptyProperties); - - broker.setRouter(new SchemaAwareRpcBroker("/",schemaService)); - + + broker.setRouter(new SchemaAwareRpcBroker("/", SchemaContextProviders.fromSchemaService(schemaService))); + dataService = new DataBrokerImpl(); dataService.setExecutor(broker.getExecutor()); - + dataReg = context.registerService(DataBrokerService, dataService, emptyProperties); dataProviderReg = context.registerService(DataProviderService, dataService, emptyProperties); wrappedStore = new SchemaAwareDataStoreAdapter(); wrappedStore.changeDelegate(store); wrappedStore.setValidationEnabled(false); - - context.registerService(SchemaServiceListener,wrappedStore,emptyProperties) - + + context.registerService(SchemaServiceListener, wrappedStore, emptyProperties) + dataService.registerConfigurationReader(ROOT, wrappedStore); dataService.registerCommitHandler(ROOT, wrappedStore); dataService.registerOperationalReader(ROOT, wrappedStore); - + mountService = new MountPointManagerImpl(); mountService.setDataBroker(dataService); - + mountReg = context.registerService(MountService, mountService, emptyProperties); - mountProviderReg = context.registerService(MountProvisionService, mountService, emptyProperties); + mountProviderReg = context.registerService(MountProvisionService, mountService, emptyProperties); } override def close() { @@ -77,5 +79,5 @@ class BrokerConfigActivator implements AutoCloseable { mountReg?.unregister(); mountProviderReg?.unregister(); } - -} \ No newline at end of file + +} diff --git a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/SchemaServiceImpl.java b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/SchemaServiceImpl.java index 5fdd25c9e3..428521b72f 100644 --- a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/SchemaServiceImpl.java +++ b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/SchemaServiceImpl.java @@ -1,45 +1,40 @@ +/* + * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ package org.opendaylight.controller.sal.dom.broker; -import java.io.IOException; -import java.io.InputStream; +import static com.google.common.base.Preconditions.checkState; + import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; import java.util.Enumeration; -import java.util.List; -import java.util.Set; -import java.util.zip.Checksum; -import org.opendaylight.yangtools.yang.model.api.SchemaContext; -import org.osgi.util.tracker.BundleTracker; -import org.osgi.util.tracker.BundleTrackerCustomizer; -import org.osgi.util.tracker.ServiceTracker; -import org.osgi.util.tracker.ServiceTrackerCustomizer; +import org.opendaylight.controller.sal.core.api.model.SchemaService; +import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener; +import org.opendaylight.controller.sal.dom.broker.impl.SchemaContextProvider; +import org.opendaylight.yangtools.concepts.ListenerRegistration; +import org.opendaylight.yangtools.concepts.Registration; +import org.opendaylight.yangtools.concepts.util.ListenerRegistry; import org.opendaylight.yangtools.yang.model.api.Module; -import org.opendaylight.yangtools.yang.model.parser.api.YangModelParser; -import org.opendaylight.yangtools.yang.parser.impl.YangParserImpl; +import org.opendaylight.yangtools.yang.model.api.SchemaContext; +import org.opendaylight.yangtools.yang.parser.impl.util.URLSchemaContextResolver; import org.osgi.framework.Bundle; -import org.osgi.framework.BundleActivator; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleEvent; import org.osgi.framework.ServiceReference; -import org.opendaylight.yangtools.concepts.ListenerRegistration; -import org.opendaylight.yangtools.concepts.util.ListenerRegistry; -import org.opendaylight.controller.sal.core.api.model.SchemaService; -import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener; -import org.opendaylight.controller.sal.dom.broker.impl.SchemaContextProvider; +import org.osgi.util.tracker.BundleTracker; +import org.osgi.util.tracker.BundleTrackerCustomizer; +import org.osgi.util.tracker.ServiceTracker; +import org.osgi.util.tracker.ServiceTrackerCustomizer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.collect.Collections2; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.Sets; - -import static com.google.common.base.Preconditions.*; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSet.Builder; public class SchemaServiceImpl implements // SchemaContextProvider, // @@ -49,22 +44,18 @@ public class SchemaServiceImpl implements // private static final Logger logger = LoggerFactory.getLogger(SchemaServiceImpl.class); private ListenerRegistry listeners; - private YangModelParser parser; - + private BundleContext context; private BundleScanner scanner = new BundleScanner(); - /** - * Map of currently problematic yang files that should get fixed eventually - * after all events are received. - */ - private final Multimap inconsistentBundlesToYangURLs = HashMultimap.create(); - private final Multimap consistentBundlesToYangURLs = HashMultimap.create(); - private BundleTracker bundleTracker; - private final YangStoreCache cache = new YangStoreCache(); + private BundleTracker>> bundleTracker; + + private final URLSchemaContextResolver contextResolver = new URLSchemaContextResolver(); private ServiceTracker listenerTracker; + private boolean starting = true; + public ListenerRegistry getListeners() { return listeners; } @@ -73,14 +64,6 @@ public class SchemaServiceImpl implements // this.listeners = listeners; } - public YangModelParser getParser() { - return parser; - } - - public void setParser(YangModelParser parser) { - this.parser = parser; - } - public BundleContext getContext() { return context; } @@ -90,53 +73,41 @@ public class SchemaServiceImpl implements // } public void start() { - checkState(parser != null); checkState(context != null); if (listeners == null) { listeners = new ListenerRegistry<>(); } - listenerTracker = new ServiceTracker<>(context, SchemaServiceListener.class, this); - bundleTracker = new BundleTracker(context, BundleEvent.RESOLVED | BundleEvent.UNRESOLVED, scanner); + listenerTracker = new ServiceTracker<>(context, SchemaServiceListener.class, SchemaServiceImpl.this); + bundleTracker = new BundleTracker>>(context, BundleEvent.RESOLVED + | BundleEvent.UNRESOLVED, scanner); bundleTracker.open(); listenerTracker.open(); + starting = false; + tryToUpdateSchemaContext(); } - @Override public SchemaContext getSchemaContext() { return getGlobalContext(); } - - public SchemaContext getGlobalContext() { - return getSchemaContextSnapshot(); - } - public synchronized SchemaContext getSchemaContextSnapshot() { - Optional yangStoreOpt = cache.getCachedSchemaContext(consistentBundlesToYangURLs); - if (yangStoreOpt.isPresent()) { - return yangStoreOpt.get(); - } - SchemaContext snapshot = createSnapshot(parser, consistentBundlesToYangURLs); - updateCache(snapshot); - return snapshot; + public SchemaContext getGlobalContext() { + return contextResolver.getSchemaContext().orNull(); } @Override public void addModule(Module module) { - // TODO Auto-generated method stub throw new UnsupportedOperationException(); } @Override public SchemaContext getSessionContext() { - // TODO Auto-generated method stub throw new UnsupportedOperationException(); } @Override public void removeModule(Module module) { - // TODO Auto-generated method stub throw new UnsupportedOperationException(); } @@ -147,63 +118,17 @@ public class SchemaServiceImpl implements // @Override public void close() throws Exception { - bundleTracker.close(); - // FIXME: Add listeners.close(); - - } - - private synchronized boolean tryToUpdateState(Collection changedURLs, Multimap proposedNewState, - boolean adding) { - Preconditions.checkArgument(!changedURLs.isEmpty(), "No change can occur when no URLs are changed"); - - try { - // consistent state - // merge into - SchemaContext snapshot = createSnapshot(parser, proposedNewState); - consistentBundlesToYangURLs.clear(); - consistentBundlesToYangURLs.putAll(proposedNewState); - inconsistentBundlesToYangURLs.clear(); - // update cache - updateCache(snapshot); - logger.trace("SchemaService updated to new consistent state"); - logger.trace("SchemaService updated to new consistent state containing {}", consistentBundlesToYangURLs); - - // notifyListeners(changedURLs, adding); - return true; - } catch (Exception e) { - // inconsistent state - logger.debug( - "SchemaService is falling back on last consistent state containing {}, inconsistent yang files {}", - consistentBundlesToYangURLs, inconsistentBundlesToYangURLs, e); - return false; + if (bundleTracker != null) { + bundleTracker.close(); } + if (listenerTracker != null) { + listenerTracker.close(); + } + // FIXME: Add listeners.close(); } - private static Collection fromUrlsToInputStreams(Multimap multimap) { - return Collections2.transform(multimap.values(), new Function() { - - @Override - public InputStream apply(URL url) { - try { - return url.openStream(); - } catch (IOException e) { - logger.warn("Unable to open stream from {}", url); - throw new IllegalStateException("Unable to open stream from " + url, e); - } - } - }); - } - - private static SchemaContext createSnapshot(YangModelParser parser, Multimap multimap) { - List models = new ArrayList<>(fromUrlsToInputStreams(multimap)); - Set modules = parser.parseYangModelsFromStreams(models); - SchemaContext yangStoreSnapshot = parser.resolveSchemaContext(modules); - return yangStoreSnapshot; - } - - private void updateCache(SchemaContext snapshot) { - cache.cacheYangStore(consistentBundlesToYangURLs, snapshot); + private void updateContext(SchemaContext snapshot) { Object[] services = listenerTracker.getServices(); if (services != null) { for (Object rawListener : services) { @@ -224,44 +149,30 @@ public class SchemaServiceImpl implements // } } - private class BundleScanner implements BundleTrackerCustomizer { + private class BundleScanner implements BundleTrackerCustomizer>> { @Override - public Object addingBundle(Bundle bundle, BundleEvent event) { + public ImmutableSet> addingBundle(Bundle bundle, BundleEvent event) { - // Ignore system bundle: - // system bundle might have config-api on classpath && - // config-api contains yang files => - // system bundle might contain yang files from that bundle if (bundle.getBundleId() == 0) { - return bundle; + return ImmutableSet.of(); } Enumeration enumeration = bundle.findEntries("META-INF/yang", "*.yang", false); - if (enumeration != null && enumeration.hasMoreElements()) { - synchronized (this) { - List addedURLs = new ArrayList<>(); - while (enumeration.hasMoreElements()) { - URL url = enumeration.nextElement(); - addedURLs.add(url); - } - logger.trace("Bundle {} has event {}, bundle state {}, URLs {}", bundle, event, bundle.getState(), - addedURLs); - // test that yang store is consistent - Multimap proposedNewState = HashMultimap.create(consistentBundlesToYangURLs); - proposedNewState.putAll(inconsistentBundlesToYangURLs); - proposedNewState.putAll(bundle, addedURLs); - boolean adding = true; - - if (tryToUpdateState(addedURLs, proposedNewState, adding) == false) { - inconsistentBundlesToYangURLs.putAll(bundle, addedURLs); - } - } + Builder> builder = ImmutableSet.> builder(); + while (enumeration != null && enumeration.hasMoreElements()) { + Registration reg = contextResolver.registerSource(enumeration.nextElement()); + builder.add(reg); + } + ImmutableSet> urls = builder.build(); + if(urls.isEmpty()) { + return urls; } - return bundle; + tryToUpdateSchemaContext(); + return urls; } @Override - public void modifiedBundle(Bundle bundle, BundleEvent event, Object object) { + public void modifiedBundle(Bundle bundle, BundleEvent event, ImmutableSet> object) { logger.debug("Modified bundle {} {} {}", bundle, event, object); } @@ -272,41 +183,15 @@ public class SchemaServiceImpl implements // */ @Override - public synchronized void removedBundle(Bundle bundle, BundleEvent event, Object object) { - inconsistentBundlesToYangURLs.removeAll(bundle); - Collection consistentURLsToBeRemoved = consistentBundlesToYangURLs.removeAll(bundle); - - if (consistentURLsToBeRemoved.isEmpty()) { - return; // no change - } - boolean adding = false; - // notifyListeners(consistentURLsToBeRemoved, adding); - } - } - - private static final class YangStoreCache { - - Set cachedUrls; - SchemaContext cachedContextSnapshot; - - Optional getCachedSchemaContext(Multimap bundlesToYangURLs) { - Set urls = setFromMultimapValues(bundlesToYangURLs); - if (cachedUrls != null && cachedUrls.equals(urls)) { - Preconditions.checkState(cachedContextSnapshot != null); - return Optional.of(cachedContextSnapshot); + public synchronized void removedBundle(Bundle bundle, BundleEvent event, ImmutableSet> urls) { + for (Registration url : urls) { + try { + url.close(); + } catch (Exception e) { + e.printStackTrace(); + } } - return Optional.absent(); - } - - private static Set setFromMultimapValues(Multimap bundlesToYangURLs) { - Set urls = Sets.newHashSet(bundlesToYangURLs.values()); - Preconditions.checkState(bundlesToYangURLs.size() == urls.size()); - return urls; - } - - void cacheYangStore(Multimap urls, SchemaContext ctx) { - this.cachedUrls = setFromMultimapValues(urls); - this.cachedContextSnapshot = ctx; + tryToUpdateSchemaContext(); } } @@ -321,6 +206,16 @@ public class SchemaServiceImpl implements // return listener; } + public synchronized void tryToUpdateSchemaContext() { + if(starting ) { + return; + } + Optional schema = contextResolver.tryToUpdateSchemaContext(); + if(schema.isPresent()) { + updateContext(schema.get()); + } + } + @Override public void modifiedService(ServiceReference reference, SchemaServiceListener service) { // NOOP diff --git a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProvider.java b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProvider.java index 3cf9a5dabc..fc8ccd6746 100644 --- a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProvider.java +++ b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProvider.java @@ -1,11 +1,16 @@ +/* + * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ package org.opendaylight.controller.sal.dom.broker.impl; import org.opendaylight.yangtools.yang.model.api.SchemaContext; -import com.google.common.base.Optional; - public interface SchemaContextProvider { SchemaContext getSchemaContext(); - + } diff --git a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProviders.java b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProviders.java new file mode 100644 index 0000000000..df985cb404 --- /dev/null +++ b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProviders.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2013 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.sal.dom.broker.impl; + +import org.opendaylight.controller.sal.core.api.model.SchemaService; +import org.opendaylight.yangtools.concepts.Delegator; +import org.opendaylight.yangtools.yang.model.api.SchemaContext; + +public class SchemaContextProviders { + + public static final SchemaContextProvider fromSchemaService(SchemaService schemaService) { + if (schemaService instanceof SchemaContextProvider) { + return (SchemaContextProvider) schemaService; + } + return new SchemaServiceAdapter(schemaService); + } + + private final static class SchemaServiceAdapter implements SchemaContextProvider, Delegator { + + private final SchemaService service; + + public SchemaServiceAdapter(SchemaService service) { + super(); + this.service = service; + } + + @Override + public SchemaContext getSchemaContext() { + return service.getGlobalContext(); + } + + @Override + public SchemaService getDelegate() { + return service; + } + + @Override + public String toString() { + return "SchemaServiceAdapter [service=" + service + "]"; + } + } +}