Ensure CSS modules are closed before blueprint containers on shutdown 01/40801/4
authorTom Pantelis <tpanteli@brocade.com>
Thu, 23 Jun 2016 14:35:07 +0000 (10:35 -0400)
committerTom Pantelis <tpanteli@brocade.com>
Thu, 21 Jul 2016 14:25:48 +0000 (14:25 +0000)
Change-Id: I9be36a819423e904030540b161437b6f2ffd091d
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>
karaf/opendaylight-karaf-resources/src/main/resources/etc/custom.properties
opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintBundleTracker.java
opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java
opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java [new file with mode: 0644]
opendaylight/config/config-manager/src/main/java/org/opendaylight/controller/config/manager/impl/osgi/ConfigManagerActivator.java

index 56ba06ecaf6bc15fb8492299d192a04e224e3f0a..3eddcf38b33e9e69baa7eeb7fd82362304bf5e39 100644 (file)
@@ -20,6 +20,13 @@ karaf.delay.console=true
 # Set security provider to BouncyCastle
 org.apache.karaf.security.providers = org.bouncycastle.jce.provider.BouncyCastleProvider
 
+# We set this to false to disable the Aries BlueprintExtender from doing its orderly container
+# shutdown so we can do it after the CSS has shut down all its modules. Otherwise Aries will
+# shutdown blueprint containers when the karaf framework starts shutdown (ie when bundle 0 is
+# stopped) which can cause failures on CSS module shutdown due to the core blueprint containers
+# and services already being shut down. This setting can be removed when/if CSS is removed
+# completely from ODL.
+org.apache.aries.blueprint.preemptiveShutdown=false
 
 netconf.config.persister.active=1
 
index 5727c13ba1e888825d94b96bcc1c330bdf39fa1e..6781b6514d313c5cf22ff4c29584d11fa8ca65bd 100644 (file)
@@ -7,21 +7,29 @@
  */
 package org.opendaylight.controller.blueprint;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Dictionary;
 import java.util.Enumeration;
+import java.util.HashSet;
 import java.util.Hashtable;
 import java.util.List;
 import org.apache.aries.blueprint.NamespaceHandler;
 import org.apache.aries.blueprint.services.BlueprintExtenderService;
 import org.apache.aries.util.AriesFrameworkUtil;
 import org.opendaylight.controller.blueprint.ext.OpendaylightNamespaceHandler;
+import org.opendaylight.controller.config.api.ConfigSystemService;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.BundleActivator;
 import org.osgi.framework.BundleContext;
 import org.osgi.framework.BundleEvent;
 import org.osgi.framework.ServiceReference;
 import org.osgi.framework.ServiceRegistration;
+import org.osgi.framework.SynchronousBundleListener;
+import org.osgi.service.blueprint.container.BlueprintContainer;
 import org.osgi.service.blueprint.container.EventConstants;
 import org.osgi.service.event.Event;
 import org.osgi.service.event.EventHandler;
@@ -40,15 +48,20 @@ import org.slf4j.LoggerFactory;
  *
  * @author Thomas Pantelis
  */
-public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCustomizer<Bundle>, EventHandler {
+public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCustomizer<Bundle>, EventHandler,
+        SynchronousBundleListener {
     private static final Logger LOG = LoggerFactory.getLogger(BlueprintBundleTracker.class);
     private static final String BLUEPRINT_FILE_PATH = "org/opendaylight/blueprint/";
     private static final String BLUEPRINT_FLE_PATTERN = "*.xml";
+    private static final long SYSTEM_BUNDLE_ID = 0;
 
     private ServiceTracker<BlueprintExtenderService, BlueprintExtenderService> serviceTracker;
     private BundleTracker<Bundle> bundleTracker;
+    private BundleContext bundleContext;
     private volatile BlueprintExtenderService blueprintExtenderService;
     private volatile ServiceRegistration<?> blueprintContainerRestartReg;
+    private volatile BlueprintContainerRestartServiceImpl restartService;
+    private volatile boolean shuttingDown;
     private ServiceRegistration<?> eventHandlerReg;
     private ServiceRegistration<?> namespaceReg;
 
@@ -59,6 +72,8 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus
     public void start(BundleContext context) {
         LOG.info("Starting {}", getClass().getSimpleName());
 
+        bundleContext = context;
+
         registerBlueprintEventHandler(context);
 
         registerNamespaceHandler(context);
@@ -73,11 +88,13 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus
                         blueprintExtenderService = reference.getBundle().getBundleContext().getService(reference);
                         bundleTracker.open();
 
+                        context.addBundleListener(BlueprintBundleTracker.this);
+
                         LOG.debug("Got BlueprintExtenderService");
 
+                        restartService = new BlueprintContainerRestartServiceImpl(blueprintExtenderService);
                         blueprintContainerRestartReg = context.registerService(
-                                BlueprintContainerRestartService.class.getName(),
-                                new BlueprintContainerRestartServiceImpl(blueprintExtenderService), new Hashtable<>());
+                                BlueprintContainerRestartService.class.getName(), restartService, new Hashtable<>());
 
                         return blueprintExtenderService;
                     }
@@ -121,6 +138,18 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus
         AriesFrameworkUtil.safeUnregisterService(blueprintContainerRestartReg);
     }
 
+    /**
+     * Implemented from SynchronousBundleListener.
+     */
+    @Override
+    public void bundleChanged(BundleEvent event) {
+        // If the system bundle (id 0) is stopping, do an orderly shutdown of all blueprint containers. On
+        // shutdown the system bundle is stopped first.
+        if(event.getBundle().getBundleId() == SYSTEM_BUNDLE_ID && event.getType() == BundleEvent.STOPPING) {
+            shutdownAllContainers();
+        }
+    }
+
     /**
      * Implemented from BundleActivator.
      */
@@ -135,6 +164,10 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus
      */
     @Override
     public void modifiedBundle(Bundle bundle, BundleEvent event, Bundle object) {
+        if(shuttingDown) {
+            return;
+        }
+
         if(bundle.getState() == Bundle.ACTIVE) {
             List<Object> paths = findBlueprintPaths(bundle);
 
@@ -176,4 +209,135 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus
             return Collections.list((Enumeration)e);
         }
     }
+
+    private void shutdownAllContainers() {
+        shuttingDown = true;
+
+        restartService.close();
+
+        // Close all CSS modules first.
+        ConfigSystemService configSystem = getOSGiService(ConfigSystemService.class);
+        if(configSystem != null) {
+            configSystem.closeAllConfigModules();
+        }
+
+        LOG.info("Shutting down all blueprint containers...");
+
+        Collection<Bundle> containerBundles = new HashSet<>(Arrays.asList(bundleContext.getBundles()));
+        while(!containerBundles.isEmpty()) {
+            // For each iteration of getBundlesToDestroy, as containers are destroyed, other containers become
+            // eligible to be destroyed. We loop until we've destroyed them all.
+            for(Bundle bundle : getBundlesToDestroy(containerBundles)) {
+                containerBundles.remove(bundle);
+                BlueprintContainer container = blueprintExtenderService.getContainer(bundle);
+                if(container != null) {
+                    blueprintExtenderService.destroyContainer(bundle, container);
+                }
+            }
+        }
+
+        LOG.info("Shutdown of blueprint containers complete");
+    }
+
+    private List<Bundle> getBundlesToDestroy(Collection<Bundle> containerBundles) {
+        List<Bundle> bundlesToDestroy = new ArrayList<Bundle>();
+
+        // Find all container bundles that either have no registered services or whose services are no
+        // longer in use.
+        for(Bundle bundle : containerBundles) {
+            ServiceReference<?>[] references = bundle.getRegisteredServices();
+            int usage = 0;
+            if(references != null) {
+                for(ServiceReference<?> reference : references) {
+                    usage += getServiceUsage(reference);
+                }
+            }
+
+            LOG.debug("Usage for bundle {} is {}", bundle, usage);
+            if(usage == 0) {
+                bundlesToDestroy.add(bundle);
+            }
+        }
+
+        if(!bundlesToDestroy.isEmpty()) {
+            Collections.sort(bundlesToDestroy, new Comparator<Bundle>() {
+                @Override
+                public int compare(Bundle b1, Bundle b2) {
+                    return (int) (b2.getLastModified() - b1.getLastModified());
+                }
+            });
+
+            LOG.debug("Selected bundles {} for destroy (no services in use)", bundlesToDestroy);
+        } else {
+            // There's either no more container bundles or they all have services being used. For
+            // the latter it means there's either circular service usage or a service is being used
+            // by a non-container bundle. But we need to make progress so we pick the bundle with a
+            // used service with the highest service ID. Each service is assigned a monotonically
+            // increasing ID as they are registered. By picking the bundle with the highest service
+            // ID, we're picking the bundle that was (likely) started after all the others and thus
+            // is likely the safest to destroy at this point.
+
+            ServiceReference<?> ref = null;
+            for(Bundle bundle : containerBundles) {
+                ServiceReference<?>[] references = bundle.getRegisteredServices();
+                if(references == null) {
+                    continue;
+                }
+
+                for(ServiceReference<?> reference : references) {
+                    // We did check the service usage above but it's possible the usage has changed since
+                    // then,
+                    if(getServiceUsage(reference) == 0) {
+                        continue;
+                    }
+
+                    // Choose 'reference' if it has a lower service ranking or, if the rankings are equal
+                    // which is usually the case, if it has a higher service ID. For the latter the < 0
+                    // check looks backwards but that's how ServiceReference#compareTo is documented to work.
+                    if(ref == null || reference.compareTo(ref) < 0) {
+                        LOG.debug("Currently selecting bundle {} for destroy (with reference {})", bundle, reference);
+                        ref = reference;
+                    }
+                }
+            }
+
+            if(ref != null) {
+                bundlesToDestroy.add(ref.getBundle());
+            }
+
+            LOG.debug("Selected bundle {} for destroy (lowest ranking service or highest service ID)",
+                    bundlesToDestroy);
+        }
+
+        return bundlesToDestroy;
+    }
+
+    private static int getServiceUsage(ServiceReference<?> ref) {
+        Bundle[] usingBundles = ref.getUsingBundles();
+        return usingBundles != null ? usingBundles.length : 0;
+    }
+
+    private <T> T getOSGiService(Class<T> serviceInterface) {
+        try {
+            ServiceReference<T> serviceReference = bundleContext.getServiceReference(serviceInterface);
+            if(serviceReference == null) {
+                LOG.warn("{} service reference not found", serviceInterface.getSimpleName());
+                return null;
+            }
+
+            T service = bundleContext.getService(serviceReference);
+            if(service == null) {
+                // This could happen on shutdown if the service was already unregistered so we log as debug.
+                LOG.debug("{} service instance was not found", serviceInterface.getSimpleName());
+            }
+
+            return service;
+        } catch(IllegalStateException e) {
+            // This is thrown if the BundleContext is no longer valid which is possible on shutdown so we
+            // log as debug.
+            LOG.debug("Error obtaining OSGi service {}", serviceInterface.getSimpleName(), e);
+        }
+
+        return null;
+    }
 }
index d0e39873d99b3d223e70d15f11694853214da428..e70fc372b73f7ed5e328d9d37151fb863abb7908 100644 (file)
@@ -77,6 +77,10 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo
 
     @Override
     public void restartContainerAndDependents(final Bundle bundle) {
+        if(restartExecutor.isShutdown()) {
+            return;
+        }
+
         LOG.debug("restartContainerAndDependents for bundle {}", bundle);
 
         restartExecutor.execute(new Runnable() {
diff --git a/opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java b/opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java
new file mode 100644 (file)
index 0000000..9a1470a
--- /dev/null
@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.config.api;
+
+/**
+ * Service interface for the config system.
+ *
+ * @author Thomas Pantelis
+ */
+public interface ConfigSystemService {
+    /**
+     * This method closes all the config system modules. This method should only be called on process
+     * shutdown and is provided as a hook to control the shutdown sequence.
+     */
+    void closeAllConfigModules();
+}
index ec27bc724fda542578113ecdd19074791ea30c91..781debb25668afcce080b71729044f1983ee7f7a 100644 (file)
@@ -15,6 +15,7 @@ import java.util.Collection;
 import java.util.List;
 import javax.management.InstanceAlreadyExistsException;
 import javax.management.MBeanServer;
+import org.opendaylight.controller.config.api.ConfigSystemService;
 import org.opendaylight.controller.config.api.ConfigRegistry;
 import org.opendaylight.controller.config.manager.impl.ConfigRegistryImpl;
 import org.opendaylight.controller.config.manager.impl.jmx.ConfigRegistryJMXRegistrator;
@@ -38,7 +39,7 @@ import org.osgi.util.tracker.ServiceTracker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class ConfigManagerActivator implements BundleActivator, SynchronousBundleListener {
+public class ConfigManagerActivator implements BundleActivator, SynchronousBundleListener, ConfigSystemService {
 
     private static final Logger LOG = LoggerFactory.getLogger(ConfigManagerActivator.class);
 
@@ -123,9 +124,12 @@ public class ConfigManagerActivator implements BundleActivator, SynchronousBundl
                     blankTransactionServiceTracker);
             serviceTracker.open();
 
+            AutoCloseable configMgrReg = registerService(context, this, ConfigSystemService.class);
+
             List<AutoCloseable> list = Arrays.asList(bindingContextProvider, clsReg,
                     wrap(moduleFactoryBundleTracker), moduleInfoBundleTracker,
-                    configRegReg, configRegistryJMXRegistrator, configRegistryJMXRegistratorWithNotifications, wrap(serviceTracker), moduleInfoRegistryWrapper, notifyingConfigRegistry);
+                    configRegReg, configRegistryJMXRegistrator, configRegistryJMXRegistratorWithNotifications,
+                    wrap(serviceTracker), moduleInfoRegistryWrapper, notifyingConfigRegistry, configMgrReg);
             autoCloseable = OsgiRegistrationUtil.aggregate(list);
 
             context.addBundleListener(this);
@@ -157,4 +161,11 @@ public class ConfigManagerActivator implements BundleActivator, SynchronousBundl
             configRegistry.close();
         }
     }
+
+    @Override
+    public void closeAllConfigModules() {
+        if(configRegistry != null) {
+            configRegistry.close();
+        }
+    }
 }