Restart config Modules when blueprint container restarted

Modified the BlueprintContainerRestartServiceImpl to recreate/restart
associated config system Modules when a blueprint container is
restarted. A config Module is associated to a service instance that is
advertised via blueprint via adding certain service properties that
identifies the module namespace, module name and instance name. When
these properties are encountered for a service while traversing the
service usage tree, the module info is queued. After all blueprint
containers are restarted, the config Modules are recreated/restarted.

Change-Id: Ia16830620bfb54896ebf54d0542d31a51d4b067a
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>

diff --git a/opendaylight/blueprint/pom.xml b/opendaylight/blueprint/pom.xml
index f06cade19eddeb2e79e2079204513f713cb200e8..660f6ea83c4fa2673f5097d1b67439214bf190c3 100644 (file)
--- a/opendaylight/blueprint/pom.xml
+++ b/opendaylight/blueprint/pom.xml
@@ -27,6 +27,10 @@
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>sal-binding-api</artifactId>
     </dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>sal-binding-api</artifactId>
     </dependency>

+    <dependency>
+      <groupId>org.opendaylight.controller</groupId>
+      <artifactId>config-manager-facade-xml</artifactId>
+    </dependency>

     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>sal-core-api</artifactId>
     <dependency>
       <groupId>org.opendaylight.controller</groupId>
       <artifactId>sal-core-api</artifactId>

diff --git a/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java b/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java
index 107ea39c1c6c8f4753d38dd07db6da74775b9cf5..d0e39873d99b3d223e70d15f11694853214da428 100644 (file)
--- a/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java
+++ b/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java
@@ -7,19 +7,54 @@
  */
 package org.opendaylight.controller.blueprint;
 
  */
 package org.opendaylight.controller.blueprint;
 

+import com.google.common.base.Optional;

 import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;

+import java.lang.management.ManagementFactory;
+import java.util.AbstractMap.SimpleEntry;

 import java.util.ArrayList;
 import java.util.ArrayList;

+import java.util.Dictionary;
+import java.util.Hashtable;

 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.LinkedHashSet;
 import java.util.List;

+import java.util.Map.Entry;

 import java.util.Set;
 import java.util.Set;

+import java.util.concurrent.CountDownLatch;

 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;

+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import javax.xml.parsers.ParserConfigurationException;

 import org.apache.aries.blueprint.services.BlueprintExtenderService;
 import org.apache.aries.blueprint.services.BlueprintExtenderService;

+import org.apache.aries.util.AriesFrameworkUtil;
+import org.opendaylight.controller.config.api.ConfigRegistry;
+import org.opendaylight.controller.config.api.ConflictingVersionException;
+import org.opendaylight.controller.config.api.ModuleIdentifier;
+import org.opendaylight.controller.config.api.ValidationException;
+import org.opendaylight.controller.config.facade.xml.ConfigExecution;
+import org.opendaylight.controller.config.facade.xml.ConfigSubsystemFacade;
+import org.opendaylight.controller.config.facade.xml.ConfigSubsystemFacadeFactory;
+import org.opendaylight.controller.config.facade.xml.TestOption;
+import org.opendaylight.controller.config.facade.xml.mapping.config.Config;
+import org.opendaylight.controller.config.facade.xml.strategy.EditStrategyType;
+import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
+import org.opendaylight.controller.config.util.xml.DocumentedException;
+import org.opendaylight.controller.config.util.xml.XmlElement;
+import org.opendaylight.controller.config.util.xml.XmlMappingConstants;
+import org.opendaylight.controller.config.util.xml.XmlUtil;

 import org.osgi.framework.Bundle;
 import org.osgi.framework.Bundle;

+import org.osgi.framework.BundleContext;

 import org.osgi.framework.ServiceReference;
 import org.osgi.framework.ServiceReference;

+import org.osgi.framework.ServiceRegistration;
+import org.osgi.service.blueprint.container.EventConstants;
+import org.osgi.service.event.Event;
+import org.osgi.service.event.EventHandler;

 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

+import org.w3c.dom.Document;
+import org.w3c.dom.Element;

 
 /**
  * Implementation of the BlueprintContainerRestartService.
 
 /**
  * Implementation of the BlueprintContainerRestartService.


@@ -28,6 +63,9 @@ import org.slf4j.LoggerFactory;
  */
 class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintContainerRestartService {
     private static final Logger LOG = LoggerFactory.getLogger(BlueprintContainerRestartServiceImpl.class);
  */
 class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintContainerRestartService {
     private static final Logger LOG = LoggerFactory.getLogger(BlueprintContainerRestartServiceImpl.class);

+    private static final String CONFIG_MODULE_NAMESPACE_PROP = "config-module-namespace";
+    private static final String CONFIG_MODULE_NAME_PROP = "config-module-name";
+    private static final String CONFIG_INSTANCE_NAME_PROP = "config-instance-name";

 
     private final ExecutorService restartExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().
             setDaemon(true).setNameFormat("BlueprintContainerRestartService").build());
 
     private final ExecutorService restartExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().
             setDaemon(true).setNameFormat("BlueprintContainerRestartService").build());


@@ -53,7 +91,8 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo
     private void restartContainerAndDependentsInternal(Bundle forBundle) {
         // We use a LinkedHashSet to preserve insertion order as we walk the service usage hierarchy.
         Set<Bundle> containerBundlesSet = new LinkedHashSet<>();
     private void restartContainerAndDependentsInternal(Bundle forBundle) {
         // We use a LinkedHashSet to preserve insertion order as we walk the service usage hierarchy.
         Set<Bundle> containerBundlesSet = new LinkedHashSet<>();

-        findDependentContainersRecursively(forBundle, containerBundlesSet);
+        List<Entry<String, ModuleIdentifier>> configModules = new ArrayList<>();
+        findDependentContainersRecursively(forBundle, containerBundlesSet, configModules);

 
         List<Bundle> containerBundles = new ArrayList<>(containerBundlesSet);
 
 
         List<Bundle> containerBundles = new ArrayList<>(containerBundlesSet);
 


@@ -65,6 +104,21 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo
             blueprintExtenderService.destroyContainer(bundle, blueprintExtenderService.getContainer(bundle));
         }
 
             blueprintExtenderService.destroyContainer(bundle, blueprintExtenderService.getContainer(bundle));
         }
 

+        // The blueprint containers are created asynchronously so we register a handler for blueprint events
+        // that are sent when a container is complete, successful or not. The CountDownLatch tells when all
+        // containers are complete. This is done to ensure all blueprint containers are finished before we
+        // restart config modules.
+        final CountDownLatch containerCreationComplete = new CountDownLatch(containerBundles.size());
+        ServiceRegistration<?> eventHandlerReg = registerEventHandler(forBundle.getBundleContext(), new EventHandler() {
+            @Override
+            public void handleEvent(Event event) {
+                LOG.debug("handleEvent {} for bundle {}", event.getTopic(), event.getProperty(EventConstants.BUNDLE));
+                if(containerBundles.contains(event.getProperty(EventConstants.BUNDLE))) {
+                    containerCreationComplete.countDown();
+                }
+            }
+        });
+

         // Restart the containers top-down starting with 'forBundle'.
         for(Bundle bundle: containerBundles) {
             List<Object> paths = BlueprintBundleTracker.findBlueprintPaths(bundle);
         // Restart the containers top-down starting with 'forBundle'.
         for(Bundle bundle: containerBundles) {
             List<Object> paths = BlueprintBundleTracker.findBlueprintPaths(bundle);


@@ -73,16 +127,102 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo
 
             blueprintExtenderService.createContainer(bundle, paths);
         }
 
             blueprintExtenderService.createContainer(bundle, paths);
         }

+
+        try {
+            containerCreationComplete.await(5, TimeUnit.MINUTES);
+        } catch(InterruptedException e) {
+            LOG.debug("CountDownLatch await was interrupted - returning");
+            return;
+        }
+
+        AriesFrameworkUtil.safeUnregisterService(eventHandlerReg);
+
+        // Now restart any associated config system Modules.
+        restartConfigModules(forBundle.getBundleContext(), configModules);
+    }
+
+    private void restartConfigModules(BundleContext bundleContext, List<Entry<String, ModuleIdentifier>> configModules) {
+        if(configModules.isEmpty()) {
+            return;
+        }
+
+        ServiceReference<ConfigSubsystemFacadeFactory> configFacadeFactoryRef =
+                bundleContext.getServiceReference(ConfigSubsystemFacadeFactory.class);
+        if(configFacadeFactoryRef == null) {
+            LOG.debug("ConfigSubsystemFacadeFactory service reference not found");
+            return;
+        }
+
+        ConfigSubsystemFacadeFactory configFacadeFactory = bundleContext.getService(configFacadeFactoryRef);
+        if(configFacadeFactory == null) {
+            LOG.debug("ConfigSubsystemFacadeFactory service not found");
+            return;
+        }
+
+        ConfigSubsystemFacade configFacade = configFacadeFactory.createFacade("BlueprintContainerRestartService");
+        try {
+            restartConfigModules(configModules, configFacade);
+        } catch(Exception e) {
+            LOG.error("Error restarting config modules", e);
+        } finally {
+            configFacade.close();
+            bundleContext.ungetService(configFacadeFactoryRef);
+        }
+
+    }
+
+    private void restartConfigModules(List<Entry<String, ModuleIdentifier>> configModules,
+            ConfigSubsystemFacade configFacade) throws ParserConfigurationException, DocumentedException,
+                    ValidationException, ConflictingVersionException {
+
+        Document document = XmlUtil.newDocument();
+        Element dataElement = XmlUtil.createElement(document, XmlMappingConstants.DATA_KEY, Optional.<String>absent());
+        Element modulesElement = XmlUtil.createElement(document, XmlMappingConstants.MODULES_KEY,
+                Optional.of(XmlMappingConstants.URN_OPENDAYLIGHT_PARAMS_XML_NS_YANG_CONTROLLER_CONFIG));
+        dataElement.appendChild(modulesElement);
+
+        Config configMapping = configFacade.getConfigMapping();
+
+        ConfigRegistry configRegistryClient = new ConfigRegistryJMXClient(ManagementFactory.getPlatformMBeanServer());
+        for(Entry<String, ModuleIdentifier> entry: configModules) {
+            String moduleNamespace = entry.getKey();
+            ModuleIdentifier moduleId = entry.getValue();
+            try {
+                ObjectName instanceON = configRegistryClient.lookupConfigBean(moduleId.getFactoryName(),
+                        moduleId.getInstanceName());
+
+                LOG.debug("Found config module instance ObjectName: {}", instanceON);
+
+                Element moduleElement = configMapping.moduleToXml(moduleNamespace, moduleId.getFactoryName(),
+                        moduleId.getInstanceName(), instanceON, document);
+                modulesElement.appendChild(moduleElement);
+            } catch(InstanceNotFoundException e) {
+                LOG.warn("Error looking up config module: namespace {}, module name {}, instance {}",
+                        moduleNamespace, moduleId.getFactoryName(), moduleId.getInstanceName(), e);
+            }
+        }
+
+        if(LOG.isDebugEnabled()) {
+            LOG.debug("Pushing config xml: {}", XmlUtil.toString(dataElement));
+        }
+
+        ConfigExecution execution = new ConfigExecution(configMapping, XmlElement.fromDomElement(dataElement),
+                TestOption.testThenSet, EditStrategyType.recreate);
+        configFacade.executeConfigExecution(execution);
+        configFacade.commitSilentTransaction();

     }
 
     /**
      * Recursively finds the services registered by the given bundle and the bundles using those services.
     }
 
     /**
      * Recursively finds the services registered by the given bundle and the bundles using those services.

-     * User bundles that have an associated blueprint container are added to containerBundles.
+     * User bundles that have an associated blueprint container are added to containerBundles. In addition,
+     * if a registered service has an associated config system Module, as determined via the presence of
+     * certain service properties, the ModuleIdentifier is added to the configModules list.

      *
      * @param bundle the bundle to traverse
      * @param containerBundles the current set of bundles containing blueprint containers
      */
      *
      * @param bundle the bundle to traverse
      * @param containerBundles the current set of bundles containing blueprint containers
      */

-    private void findDependentContainersRecursively(Bundle bundle, Set<Bundle> containerBundles) {
+    private void findDependentContainersRecursively(Bundle bundle, Set<Bundle> containerBundles,
+            List<Entry<String, ModuleIdentifier>> configModules) {

         if(!containerBundles.add(bundle)) {
             // Already seen this bundle...
             return;
         if(!containerBundles.add(bundle)) {
             // Already seen this bundle...
             return;


@@ -91,11 +231,13 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo
         ServiceReference<?>[] references = bundle.getRegisteredServices();
         if (references != null) {
             for (ServiceReference<?> reference : references) {
         ServiceReference<?>[] references = bundle.getRegisteredServices();
         if (references != null) {
             for (ServiceReference<?> reference : references) {

+                possiblyAddConfigModuleIdentifier(reference, configModules);
+

                 Bundle[] usingBundles = reference.getUsingBundles();
                 if(usingBundles != null) {
                     for(Bundle usingBundle: usingBundles) {
                         if(blueprintExtenderService.getContainer(usingBundle) != null) {
                 Bundle[] usingBundles = reference.getUsingBundles();
                 if(usingBundles != null) {
                     for(Bundle usingBundle: usingBundles) {
                         if(blueprintExtenderService.getContainer(usingBundle) != null) {

-                            findDependentContainersRecursively(usingBundle, containerBundles);
+                            findDependentContainersRecursively(usingBundle, containerBundles, configModules);

                         }
                     }
                 }
                         }
                     }
                 }


@@ -103,6 +245,48 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo
         }
     }
 
         }
     }
 

+    private void possiblyAddConfigModuleIdentifier(ServiceReference<?> reference,
+            List<Entry<String, ModuleIdentifier>> configModules) {
+        Object moduleNamespace = reference.getProperty(CONFIG_MODULE_NAMESPACE_PROP);
+        if(moduleNamespace == null) {
+            return;
+        }
+
+        String moduleName = getRequiredConfigModuleProperty(CONFIG_MODULE_NAME_PROP, moduleNamespace,
+                reference);
+        String instanceName = getRequiredConfigModuleProperty(CONFIG_INSTANCE_NAME_PROP, moduleNamespace,
+                reference);
+        if(moduleName == null || instanceName == null) {
+            return;
+        }
+
+        LOG.debug("Found service with config module: namespace {}, module name {}, instance {}",
+                moduleNamespace, moduleName, instanceName);
+
+        configModules.add(new SimpleEntry<>(moduleNamespace.toString(),
+                new ModuleIdentifier(moduleName, instanceName)));
+    }
+
+    @Nullable
+    private String getRequiredConfigModuleProperty(String propName, Object moduleNamespace,
+            ServiceReference<?> reference) {
+        Object value = reference.getProperty(propName);
+        if(value == null) {
+            LOG.warn("OSGi service with {} property is missing property {} therefore the config module can't be restarted",
+                    CONFIG_MODULE_NAMESPACE_PROP, propName);
+            return null;
+        }
+
+        return value.toString();
+    }
+
+    private ServiceRegistration<?> registerEventHandler(BundleContext bundleContext, EventHandler handler) {
+        Dictionary<String, Object> props = new Hashtable<>();
+        props.put(org.osgi.service.event.EventConstants.EVENT_TOPIC,
+                new String[]{EventConstants.TOPIC_CREATED, EventConstants.TOPIC_FAILURE});
+        return bundleContext.registerService(EventHandler.class.getName(), handler, props);
+    }
+

     @Override
     public void close() {
         restartExecutor.shutdownNow();
     @Override
     public void close() {
         restartExecutor.shutdownNow();