Introduced advanced strategy for parsing of YANG schemas 85/4585/4
authorTony Tkacik <ttkacik@cisco.com>
Wed, 22 Jan 2014 14:54:00 +0000 (15:54 +0100)
committerEd Warnicke <eaw@cisco.com>
Thu, 23 Jan 2014 01:04:29 +0000 (01:04 +0000)
  - This strategy increases convergence time, because it does not
    parse files with unsatisfied dependencies.

  - New strategy for schema parsing in runtime uses two separate parser
     Minimalistic - is used when YANG source is discovered and scans only
     for module name, revision and dependency information
     Full - parses set of YANG sources and creates model representation.

     When bundle is added / removed - tracker gets yang sources,
     registers it to SchemaResolver, which immediatelly uses
     Minimalistic parser to obtain base information.

     Then context of all known sources and their dependencies is sorted
     and filtered only to use sources which has their dependencies resolved.

     Full parser is then invoked only for models, which have dependencies
     resolved.

Change-Id: Ic62fbd86929866a21f4dbd3c62e73a02d423d7bf
Signed-off-by: Tony Tkacik <ttkacik@cisco.com>
opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/config/yang/md/sal/dom/impl/SchemaServiceImplSingletonModule.java
opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/BrokerConfigActivator.xtend
opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/SchemaServiceImpl.java
opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProvider.java
opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProviders.java [new file with mode: 0644]

index 6b597de..d61d709 100644 (file)
@@ -51,7 +51,6 @@ public final class SchemaServiceImplSingletonModule extends org.opendaylight.con
     public java.lang.AutoCloseable createInstance() {
         SchemaServiceImpl newInstance = new SchemaServiceImpl();
         newInstance.setContext(getBundleContext());
-        newInstance.setParser(new YangParserImpl());
         newInstance.start();
         return newInstance;
     }
index 3baae04..b5737a5 100644 (file)
@@ -1,73 +1,75 @@
+/*
+ * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
 package org.opendaylight.controller.sal.dom.broker
 
-import org.osgi.framework.ServiceRegistration
-import org.opendaylight.controller.sal.core.api.model.SchemaService
+import java.util.Hashtable
 import org.opendaylight.controller.sal.core.api.data.DataBrokerService
 import org.opendaylight.controller.sal.core.api.data.DataProviderService
-import org.opendaylight.controller.sal.dom.broker.impl.HashMapDataStore
+import org.opendaylight.controller.sal.core.api.data.DataStore
+import org.opendaylight.controller.sal.core.api.model.SchemaService
+import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener
 import org.opendaylight.controller.sal.core.api.mount.MountProvisionService
 import org.opendaylight.controller.sal.core.api.mount.MountService
-import org.osgi.framework.BundleContext
-import java.util.Hashtable
-import org.opendaylight.yangtools.yang.parser.impl.YangParserImpl
-import org.opendaylight.yangtools.yang.data.api.InstanceIdentifier
-import org.opendaylight.controller.sal.core.api.data.DataStore
 import org.opendaylight.controller.sal.dom.broker.impl.SchemaAwareDataStoreAdapter
-import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener
 import org.opendaylight.controller.sal.dom.broker.impl.SchemaAwareRpcBroker
+import org.opendaylight.yangtools.yang.data.api.InstanceIdentifier
+import org.osgi.framework.BundleContext
+import org.osgi.framework.ServiceRegistration
+import org.opendaylight.controller.sal.dom.broker.impl.SchemaContextProviders
 
 class BrokerConfigActivator implements AutoCloseable {
-    
-    
+
     private static val ROOT = InstanceIdentifier.builder().toInstance();
 
     @Property
     private var DataBrokerImpl dataService;
-    
+
     private var ServiceRegistration<SchemaService> schemaReg;
     private var ServiceRegistration<DataBrokerService> dataReg;
     private var ServiceRegistration<DataProviderService> dataProviderReg;
     private var ServiceRegistration<MountService> mountReg;
     private var ServiceRegistration<MountProvisionService> mountProviderReg;
-    private var SchemaServiceImpl schemaService;
+    private var SchemaService schemaService;
     private var MountPointManagerImpl mountService;
-    
+
     SchemaAwareDataStoreAdapter wrappedStore
 
-    public def void start(BrokerImpl broker,DataStore store,BundleContext context) {
+    public def void start(BrokerImpl broker, DataStore store, BundleContext context) {
         val emptyProperties = new Hashtable<String, String>();
         broker.setBundleContext(context);
-        
-        
-        schemaService = new SchemaServiceImpl();
-        schemaService.setContext(context);
-        schemaService.setParser(new YangParserImpl());
-        schemaService.start();
+
+        val serviceRef = context.getServiceReference(SchemaService);
+        schemaService = context.getService(serviceRef);
         schemaReg = context.registerService(SchemaService, schemaService, emptyProperties);
-        
-        broker.setRouter(new SchemaAwareRpcBroker("/",schemaService));
-        
+
+        broker.setRouter(new SchemaAwareRpcBroker("/", SchemaContextProviders.fromSchemaService(schemaService)));
+
         dataService = new DataBrokerImpl();
         dataService.setExecutor(broker.getExecutor());
-        
+
         dataReg = context.registerService(DataBrokerService, dataService, emptyProperties);
         dataProviderReg = context.registerService(DataProviderService, dataService, emptyProperties);
 
         wrappedStore = new SchemaAwareDataStoreAdapter();
         wrappedStore.changeDelegate(store);
         wrappedStore.setValidationEnabled(false);
-       
-        context.registerService(SchemaServiceListener,wrappedStore,emptyProperties)  
-        
+
+        context.registerService(SchemaServiceListener, wrappedStore, emptyProperties)
+
         dataService.registerConfigurationReader(ROOT, wrappedStore);
         dataService.registerCommitHandler(ROOT, wrappedStore);
         dataService.registerOperationalReader(ROOT, wrappedStore);
-        
+
         mountService = new MountPointManagerImpl();
         mountService.setDataBroker(dataService);
-        
+
         mountReg = context.registerService(MountService, mountService, emptyProperties);
-        mountProviderReg =  context.registerService(MountProvisionService, mountService, emptyProperties);
+        mountProviderReg = context.registerService(MountProvisionService, mountService, emptyProperties);
     }
 
     override def close() {
@@ -77,5 +79,5 @@ class BrokerConfigActivator implements AutoCloseable {
         mountReg?.unregister();
         mountProviderReg?.unregister();
     }
-    
-}
\ No newline at end of file
+
+}
index 5fdd25c..428521b 100644 (file)
@@ -1,45 +1,40 @@
+/*
+ * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
 package org.opendaylight.controller.sal.dom.broker;
 
-import java.io.IOException;
-import java.io.InputStream;
+import static com.google.common.base.Preconditions.checkState;
+
 import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Enumeration;
-import java.util.List;
-import java.util.Set;
-import java.util.zip.Checksum;
 
-import org.opendaylight.yangtools.yang.model.api.SchemaContext;
-import org.osgi.util.tracker.BundleTracker;
-import org.osgi.util.tracker.BundleTrackerCustomizer;
-import org.osgi.util.tracker.ServiceTracker;
-import org.osgi.util.tracker.ServiceTrackerCustomizer;
+import org.opendaylight.controller.sal.core.api.model.SchemaService;
+import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener;
+import org.opendaylight.controller.sal.dom.broker.impl.SchemaContextProvider;
+import org.opendaylight.yangtools.concepts.ListenerRegistration;
+import org.opendaylight.yangtools.concepts.Registration;
+import org.opendaylight.yangtools.concepts.util.ListenerRegistry;
 import org.opendaylight.yangtools.yang.model.api.Module;
-import org.opendaylight.yangtools.yang.model.parser.api.YangModelParser;
-import org.opendaylight.yangtools.yang.parser.impl.YangParserImpl;
+import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+import org.opendaylight.yangtools.yang.parser.impl.util.URLSchemaContextResolver;
 import org.osgi.framework.Bundle;
-import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.BundleEvent;
 import org.osgi.framework.ServiceReference;
-import org.opendaylight.yangtools.concepts.ListenerRegistration;
-import org.opendaylight.yangtools.concepts.util.ListenerRegistry;
-import org.opendaylight.controller.sal.core.api.model.SchemaService;
-import org.opendaylight.controller.sal.core.api.model.SchemaServiceListener;
-import org.opendaylight.controller.sal.dom.broker.impl.SchemaContextProvider;
+import org.osgi.util.tracker.BundleTracker;
+import org.osgi.util.tracker.BundleTrackerCustomizer;
+import org.osgi.util.tracker.ServiceTracker;
+import org.osgi.util.tracker.ServiceTrackerCustomizer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Function;
 import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
-
-import static com.google.common.base.Preconditions.*;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet.Builder;
 
 public class SchemaServiceImpl implements //
         SchemaContextProvider, //
@@ -49,22 +44,18 @@ public class SchemaServiceImpl implements //
     private static final Logger logger = LoggerFactory.getLogger(SchemaServiceImpl.class);
 
     private ListenerRegistry<SchemaServiceListener> listeners;
-    private YangModelParser parser;
-
+    
     private BundleContext context;
     private BundleScanner scanner = new BundleScanner();
 
-    /**
-     * Map of currently problematic yang files that should get fixed eventually
-     * after all events are received.
-     */
-    private final Multimap<Bundle, URL> inconsistentBundlesToYangURLs = HashMultimap.create();
-    private final Multimap<Bundle, URL> consistentBundlesToYangURLs = HashMultimap.create();
-    private BundleTracker<Object> bundleTracker;
-    private final YangStoreCache cache = new YangStoreCache();
+    private BundleTracker<ImmutableSet<Registration<URL>>> bundleTracker;
+
+    private final URLSchemaContextResolver contextResolver = new URLSchemaContextResolver();
 
     private ServiceTracker<SchemaServiceListener, SchemaServiceListener> listenerTracker;
 
+    private boolean starting = true;
+
     public ListenerRegistry<SchemaServiceListener> getListeners() {
         return listeners;
     }
@@ -73,14 +64,6 @@ public class SchemaServiceImpl implements //
         this.listeners = listeners;
     }
 
-    public YangModelParser getParser() {
-        return parser;
-    }
-
-    public void setParser(YangModelParser parser) {
-        this.parser = parser;
-    }
-
     public BundleContext getContext() {
         return context;
     }
@@ -90,53 +73,41 @@ public class SchemaServiceImpl implements //
     }
 
     public void start() {
-        checkState(parser != null);
         checkState(context != null);
         if (listeners == null) {
             listeners = new ListenerRegistry<>();
         }
 
-        listenerTracker = new ServiceTracker<>(context, SchemaServiceListener.class, this);
-        bundleTracker = new BundleTracker<Object>(context, BundleEvent.RESOLVED | BundleEvent.UNRESOLVED, scanner);
+        listenerTracker = new ServiceTracker<>(context, SchemaServiceListener.class, SchemaServiceImpl.this);
+        bundleTracker = new BundleTracker<ImmutableSet<Registration<URL>>>(context, BundleEvent.RESOLVED
+                        | BundleEvent.UNRESOLVED, scanner);
         bundleTracker.open();
         listenerTracker.open();
+        starting = false;
+        tryToUpdateSchemaContext();
     }
 
-    
     @Override
     public SchemaContext getSchemaContext() {
         return getGlobalContext();
     }
-    
-    public SchemaContext getGlobalContext() {
-        return getSchemaContextSnapshot();
-    }
 
-    public synchronized SchemaContext getSchemaContextSnapshot() {
-        Optional<SchemaContext> yangStoreOpt = cache.getCachedSchemaContext(consistentBundlesToYangURLs);
-        if (yangStoreOpt.isPresent()) {
-            return yangStoreOpt.get();
-        }
-        SchemaContext snapshot = createSnapshot(parser, consistentBundlesToYangURLs);
-        updateCache(snapshot);
-        return snapshot;
+    public SchemaContext getGlobalContext() {
+        return contextResolver.getSchemaContext().orNull();
     }
 
     @Override
     public void addModule(Module module) {
-        // TODO Auto-generated method stub
         throw new UnsupportedOperationException();
     }
 
     @Override
     public SchemaContext getSessionContext() {
-        // TODO Auto-generated method stub
         throw new UnsupportedOperationException();
     }
 
     @Override
     public void removeModule(Module module) {
-        // TODO Auto-generated method stub
         throw new UnsupportedOperationException();
     }
 
@@ -147,63 +118,17 @@ public class SchemaServiceImpl implements //
 
     @Override
     public void close() throws Exception {
-        bundleTracker.close();
-        // FIXME: Add listeners.close();
-
-    }
-
-    private synchronized boolean tryToUpdateState(Collection<URL> changedURLs, Multimap<Bundle, URL> proposedNewState,
-            boolean adding) {
-        Preconditions.checkArgument(!changedURLs.isEmpty(), "No change can occur when no URLs are changed");
-
-        try {
-            // consistent state
-            // merge into
-            SchemaContext snapshot = createSnapshot(parser, proposedNewState);
-            consistentBundlesToYangURLs.clear();
-            consistentBundlesToYangURLs.putAll(proposedNewState);
-            inconsistentBundlesToYangURLs.clear();
-            // update cache
-            updateCache(snapshot);
-            logger.trace("SchemaService updated to new consistent state");
-            logger.trace("SchemaService  updated to new consistent state containing {}", consistentBundlesToYangURLs);
-
-            // notifyListeners(changedURLs, adding);
-            return true;
-        } catch (Exception e) {
-            // inconsistent state
-            logger.debug(
-                    "SchemaService is falling back on last consistent state containing {}, inconsistent yang files {}",
-                    consistentBundlesToYangURLs, inconsistentBundlesToYangURLs, e);
-            return false;
+        if (bundleTracker != null) {
+            bundleTracker.close();
         }
+        if (listenerTracker != null) {
+            listenerTracker.close();
+        }
+        // FIXME: Add listeners.close();
     }
 
-    private static Collection<InputStream> fromUrlsToInputStreams(Multimap<Bundle, URL> multimap) {
-        return Collections2.transform(multimap.values(), new Function<URL, InputStream>() {
-
-            @Override
-            public InputStream apply(URL url) {
-                try {
-                    return url.openStream();
-                } catch (IOException e) {
-                    logger.warn("Unable to open stream from {}", url);
-                    throw new IllegalStateException("Unable to open stream from " + url, e);
-                }
-            }
-        });
-    }
-
-    private static SchemaContext createSnapshot(YangModelParser parser, Multimap<Bundle, URL> multimap) {
-        List<InputStream> models = new ArrayList<>(fromUrlsToInputStreams(multimap));
-        Set<Module> modules = parser.parseYangModelsFromStreams(models);
-        SchemaContext yangStoreSnapshot = parser.resolveSchemaContext(modules);
-        return yangStoreSnapshot;
-    }
-
-    private void updateCache(SchemaContext snapshot) {
-        cache.cacheYangStore(consistentBundlesToYangURLs, snapshot);
 
+    private void updateContext(SchemaContext snapshot) {
         Object[] services = listenerTracker.getServices();
         if (services != null) {
             for (Object rawListener : services) {
@@ -224,44 +149,30 @@ public class SchemaServiceImpl implements //
         }
     }
 
-    private class BundleScanner implements BundleTrackerCustomizer<Object> {
+    private class BundleScanner implements BundleTrackerCustomizer<ImmutableSet<Registration<URL>>> {
         @Override
-        public Object addingBundle(Bundle bundle, BundleEvent event) {
+        public ImmutableSet<Registration<URL>> addingBundle(Bundle bundle, BundleEvent event) {
 
-            // Ignore system bundle:
-            // system bundle might have config-api on classpath &&
-            // config-api contains yang files =>
-            // system bundle might contain yang files from that bundle
             if (bundle.getBundleId() == 0) {
-                return bundle;
+                return ImmutableSet.of();
             }
 
             Enumeration<URL> enumeration = bundle.findEntries("META-INF/yang", "*.yang", false);
-            if (enumeration != null && enumeration.hasMoreElements()) {
-                synchronized (this) {
-                    List<URL> addedURLs = new ArrayList<>();
-                    while (enumeration.hasMoreElements()) {
-                        URL url = enumeration.nextElement();
-                        addedURLs.add(url);
-                    }
-                    logger.trace("Bundle {} has event {}, bundle state {}, URLs {}", bundle, event, bundle.getState(),
-                            addedURLs);
-                    // test that yang store is consistent
-                    Multimap<Bundle, URL> proposedNewState = HashMultimap.create(consistentBundlesToYangURLs);
-                    proposedNewState.putAll(inconsistentBundlesToYangURLs);
-                    proposedNewState.putAll(bundle, addedURLs);
-                    boolean adding = true;
-
-                    if (tryToUpdateState(addedURLs, proposedNewState, adding) == false) {
-                        inconsistentBundlesToYangURLs.putAll(bundle, addedURLs);
-                    }
-                }
+            Builder<Registration<URL>> builder = ImmutableSet.<Registration<URL>> builder();
+            while (enumeration != null && enumeration.hasMoreElements()) {
+                Registration<URL> reg = contextResolver.registerSource(enumeration.nextElement());
+                builder.add(reg);
+            }
+            ImmutableSet<Registration<URL>> urls = builder.build();
+            if(urls.isEmpty()) {
+                return urls;
             }
-            return bundle;
+            tryToUpdateSchemaContext();
+            return urls;
         }
 
         @Override
-        public void modifiedBundle(Bundle bundle, BundleEvent event, Object object) {
+        public void modifiedBundle(Bundle bundle, BundleEvent event, ImmutableSet<Registration<URL>> object) {
             logger.debug("Modified bundle {} {} {}", bundle, event, object);
         }
 
@@ -272,41 +183,15 @@ public class SchemaServiceImpl implements //
          */
 
         @Override
-        public synchronized void removedBundle(Bundle bundle, BundleEvent event, Object object) {
-            inconsistentBundlesToYangURLs.removeAll(bundle);
-            Collection<URL> consistentURLsToBeRemoved = consistentBundlesToYangURLs.removeAll(bundle);
-
-            if (consistentURLsToBeRemoved.isEmpty()) {
-                return; // no change
-            }
-            boolean adding = false;
-            // notifyListeners(consistentURLsToBeRemoved, adding);
-        }
-    }
-
-    private static final class YangStoreCache {
-
-        Set<URL> cachedUrls;
-        SchemaContext cachedContextSnapshot;
-
-        Optional<SchemaContext> getCachedSchemaContext(Multimap<Bundle, URL> bundlesToYangURLs) {
-            Set<URL> urls = setFromMultimapValues(bundlesToYangURLs);
-            if (cachedUrls != null && cachedUrls.equals(urls)) {
-                Preconditions.checkState(cachedContextSnapshot != null);
-                return Optional.of(cachedContextSnapshot);
+        public synchronized void removedBundle(Bundle bundle, BundleEvent event, ImmutableSet<Registration<URL>> urls) {
+            for (Registration<URL> url : urls) {
+                try {
+                    url.close();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
             }
-            return Optional.absent();
-        }
-
-        private static Set<URL> setFromMultimapValues(Multimap<Bundle, URL> bundlesToYangURLs) {
-            Set<URL> urls = Sets.newHashSet(bundlesToYangURLs.values());
-            Preconditions.checkState(bundlesToYangURLs.size() == urls.size());
-            return urls;
-        }
-
-        void cacheYangStore(Multimap<Bundle, URL> urls, SchemaContext ctx) {
-            this.cachedUrls = setFromMultimapValues(urls);
-            this.cachedContextSnapshot = ctx;
+            tryToUpdateSchemaContext();
         }
     }
 
@@ -321,6 +206,16 @@ public class SchemaServiceImpl implements //
         return listener;
     }
 
+    public synchronized void tryToUpdateSchemaContext() {
+        if(starting ) {
+            return;
+        }
+        Optional<SchemaContext> schema = contextResolver.tryToUpdateSchemaContext();
+        if(schema.isPresent()) {
+            updateContext(schema.get());
+        }
+    }
+
     @Override
     public void modifiedService(ServiceReference<SchemaServiceListener> reference, SchemaServiceListener service) {
         // NOOP
index 3cf9a5d..fc8ccd6 100644 (file)
@@ -1,11 +1,16 @@
+/*
+ * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
 package org.opendaylight.controller.sal.dom.broker.impl;
 
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 
-import com.google.common.base.Optional;
-
 public interface SchemaContextProvider {
 
     SchemaContext getSchemaContext();
-    
+
 }
diff --git a/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProviders.java b/opendaylight/md-sal/sal-dom-broker/src/main/java/org/opendaylight/controller/sal/dom/broker/impl/SchemaContextProviders.java
new file mode 100644 (file)
index 0000000..df985cb
--- /dev/null
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.sal.dom.broker.impl;
+
+import org.opendaylight.controller.sal.core.api.model.SchemaService;
+import org.opendaylight.yangtools.concepts.Delegator;
+import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+
+public class SchemaContextProviders {
+
+    public static final SchemaContextProvider fromSchemaService(SchemaService schemaService) {
+        if (schemaService instanceof SchemaContextProvider) {
+            return (SchemaContextProvider) schemaService;
+        }
+        return new SchemaServiceAdapter(schemaService);
+    }
+
+    private final static class SchemaServiceAdapter implements SchemaContextProvider, Delegator<SchemaService> {
+
+        private final SchemaService service;
+
+        public SchemaServiceAdapter(SchemaService service) {
+            super();
+            this.service = service;
+        }
+
+        @Override
+        public SchemaContext getSchemaContext() {
+            return service.getGlobalContext();
+        }
+
+        @Override
+        public SchemaService getDelegate() {
+            return service;
+        }
+
+        @Override
+        public String toString() {
+            return "SchemaServiceAdapter [service=" + service + "]";
+        }
+    }
+}