From aebe2884329d666f661be53e2de3a57e4fcf61f1 Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Thu, 23 Jun 2016 10:35:07 -0400 Subject: [PATCH] Ensure CSS modules are closed before blueprint containers on shutdown Change-Id: I9be36a819423e904030540b161437b6f2ffd091d Signed-off-by: Tom Pantelis --- .../src/main/resources/etc/custom.properties | 7 + .../blueprint/BlueprintBundleTracker.java | 170 +++++++++++++++++- .../BlueprintContainerRestartServiceImpl.java | 4 + .../config/api/ConfigSystemService.java | 21 +++ .../impl/osgi/ConfigManagerActivator.java | 15 +- 5 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java diff --git a/karaf/opendaylight-karaf-resources/src/main/resources/etc/custom.properties b/karaf/opendaylight-karaf-resources/src/main/resources/etc/custom.properties index 56ba06ecaf..3eddcf38b3 100644 --- a/karaf/opendaylight-karaf-resources/src/main/resources/etc/custom.properties +++ b/karaf/opendaylight-karaf-resources/src/main/resources/etc/custom.properties @@ -20,6 +20,13 @@ karaf.delay.console=true # Set security provider to BouncyCastle org.apache.karaf.security.providers = org.bouncycastle.jce.provider.BouncyCastleProvider +# We set this to false to disable the Aries BlueprintExtender from doing its orderly container +# shutdown so we can do it after the CSS has shut down all its modules. Otherwise Aries will +# shutdown blueprint containers when the karaf framework starts shutdown (ie when bundle 0 is +# stopped) which can cause failures on CSS module shutdown due to the core blueprint containers +# and services already being shut down. This setting can be removed when/if CSS is removed +# completely from ODL. +org.apache.aries.blueprint.preemptiveShutdown=false netconf.config.persister.active=1 diff --git a/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintBundleTracker.java b/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintBundleTracker.java index 5727c13ba1..6781b6514d 100644 --- a/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintBundleTracker.java +++ b/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintBundleTracker.java @@ -7,21 +7,29 @@ */ package org.opendaylight.controller.blueprint; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Dictionary; import java.util.Enumeration; +import java.util.HashSet; import java.util.Hashtable; import java.util.List; import org.apache.aries.blueprint.NamespaceHandler; import org.apache.aries.blueprint.services.BlueprintExtenderService; import org.apache.aries.util.AriesFrameworkUtil; import org.opendaylight.controller.blueprint.ext.OpendaylightNamespaceHandler; +import org.opendaylight.controller.config.api.ConfigSystemService; import org.osgi.framework.Bundle; import org.osgi.framework.BundleActivator; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleEvent; import org.osgi.framework.ServiceReference; import org.osgi.framework.ServiceRegistration; +import org.osgi.framework.SynchronousBundleListener; +import org.osgi.service.blueprint.container.BlueprintContainer; import org.osgi.service.blueprint.container.EventConstants; import org.osgi.service.event.Event; import org.osgi.service.event.EventHandler; @@ -40,15 +48,20 @@ import org.slf4j.LoggerFactory; * * @author Thomas Pantelis */ -public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCustomizer, EventHandler { +public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCustomizer, EventHandler, + SynchronousBundleListener { private static final Logger LOG = LoggerFactory.getLogger(BlueprintBundleTracker.class); private static final String BLUEPRINT_FILE_PATH = "org/opendaylight/blueprint/"; private static final String BLUEPRINT_FLE_PATTERN = "*.xml"; + private static final long SYSTEM_BUNDLE_ID = 0; private ServiceTracker serviceTracker; private BundleTracker bundleTracker; + private BundleContext bundleContext; private volatile BlueprintExtenderService blueprintExtenderService; private volatile ServiceRegistration blueprintContainerRestartReg; + private volatile BlueprintContainerRestartServiceImpl restartService; + private volatile boolean shuttingDown; private ServiceRegistration eventHandlerReg; private ServiceRegistration namespaceReg; @@ -59,6 +72,8 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus public void start(BundleContext context) { LOG.info("Starting {}", getClass().getSimpleName()); + bundleContext = context; + registerBlueprintEventHandler(context); registerNamespaceHandler(context); @@ -73,11 +88,13 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus blueprintExtenderService = reference.getBundle().getBundleContext().getService(reference); bundleTracker.open(); + context.addBundleListener(BlueprintBundleTracker.this); + LOG.debug("Got BlueprintExtenderService"); + restartService = new BlueprintContainerRestartServiceImpl(blueprintExtenderService); blueprintContainerRestartReg = context.registerService( - BlueprintContainerRestartService.class.getName(), - new BlueprintContainerRestartServiceImpl(blueprintExtenderService), new Hashtable<>()); + BlueprintContainerRestartService.class.getName(), restartService, new Hashtable<>()); return blueprintExtenderService; } @@ -121,6 +138,18 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus AriesFrameworkUtil.safeUnregisterService(blueprintContainerRestartReg); } + /** + * Implemented from SynchronousBundleListener. + */ + @Override + public void bundleChanged(BundleEvent event) { + // If the system bundle (id 0) is stopping, do an orderly shutdown of all blueprint containers. On + // shutdown the system bundle is stopped first. + if(event.getBundle().getBundleId() == SYSTEM_BUNDLE_ID && event.getType() == BundleEvent.STOPPING) { + shutdownAllContainers(); + } + } + /** * Implemented from BundleActivator. */ @@ -135,6 +164,10 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus */ @Override public void modifiedBundle(Bundle bundle, BundleEvent event, Bundle object) { + if(shuttingDown) { + return; + } + if(bundle.getState() == Bundle.ACTIVE) { List paths = findBlueprintPaths(bundle); @@ -176,4 +209,135 @@ public class BlueprintBundleTracker implements BundleActivator, BundleTrackerCus return Collections.list((Enumeration)e); } } + + private void shutdownAllContainers() { + shuttingDown = true; + + restartService.close(); + + // Close all CSS modules first. + ConfigSystemService configSystem = getOSGiService(ConfigSystemService.class); + if(configSystem != null) { + configSystem.closeAllConfigModules(); + } + + LOG.info("Shutting down all blueprint containers..."); + + Collection containerBundles = new HashSet<>(Arrays.asList(bundleContext.getBundles())); + while(!containerBundles.isEmpty()) { + // For each iteration of getBundlesToDestroy, as containers are destroyed, other containers become + // eligible to be destroyed. We loop until we've destroyed them all. + for(Bundle bundle : getBundlesToDestroy(containerBundles)) { + containerBundles.remove(bundle); + BlueprintContainer container = blueprintExtenderService.getContainer(bundle); + if(container != null) { + blueprintExtenderService.destroyContainer(bundle, container); + } + } + } + + LOG.info("Shutdown of blueprint containers complete"); + } + + private List getBundlesToDestroy(Collection containerBundles) { + List bundlesToDestroy = new ArrayList(); + + // Find all container bundles that either have no registered services or whose services are no + // longer in use. + for(Bundle bundle : containerBundles) { + ServiceReference[] references = bundle.getRegisteredServices(); + int usage = 0; + if(references != null) { + for(ServiceReference reference : references) { + usage += getServiceUsage(reference); + } + } + + LOG.debug("Usage for bundle {} is {}", bundle, usage); + if(usage == 0) { + bundlesToDestroy.add(bundle); + } + } + + if(!bundlesToDestroy.isEmpty()) { + Collections.sort(bundlesToDestroy, new Comparator() { + @Override + public int compare(Bundle b1, Bundle b2) { + return (int) (b2.getLastModified() - b1.getLastModified()); + } + }); + + LOG.debug("Selected bundles {} for destroy (no services in use)", bundlesToDestroy); + } else { + // There's either no more container bundles or they all have services being used. For + // the latter it means there's either circular service usage or a service is being used + // by a non-container bundle. But we need to make progress so we pick the bundle with a + // used service with the highest service ID. Each service is assigned a monotonically + // increasing ID as they are registered. By picking the bundle with the highest service + // ID, we're picking the bundle that was (likely) started after all the others and thus + // is likely the safest to destroy at this point. + + ServiceReference ref = null; + for(Bundle bundle : containerBundles) { + ServiceReference[] references = bundle.getRegisteredServices(); + if(references == null) { + continue; + } + + for(ServiceReference reference : references) { + // We did check the service usage above but it's possible the usage has changed since + // then, + if(getServiceUsage(reference) == 0) { + continue; + } + + // Choose 'reference' if it has a lower service ranking or, if the rankings are equal + // which is usually the case, if it has a higher service ID. For the latter the < 0 + // check looks backwards but that's how ServiceReference#compareTo is documented to work. + if(ref == null || reference.compareTo(ref) < 0) { + LOG.debug("Currently selecting bundle {} for destroy (with reference {})", bundle, reference); + ref = reference; + } + } + } + + if(ref != null) { + bundlesToDestroy.add(ref.getBundle()); + } + + LOG.debug("Selected bundle {} for destroy (lowest ranking service or highest service ID)", + bundlesToDestroy); + } + + return bundlesToDestroy; + } + + private static int getServiceUsage(ServiceReference ref) { + Bundle[] usingBundles = ref.getUsingBundles(); + return usingBundles != null ? usingBundles.length : 0; + } + + private T getOSGiService(Class serviceInterface) { + try { + ServiceReference serviceReference = bundleContext.getServiceReference(serviceInterface); + if(serviceReference == null) { + LOG.warn("{} service reference not found", serviceInterface.getSimpleName()); + return null; + } + + T service = bundleContext.getService(serviceReference); + if(service == null) { + // This could happen on shutdown if the service was already unregistered so we log as debug. + LOG.debug("{} service instance was not found", serviceInterface.getSimpleName()); + } + + return service; + } catch(IllegalStateException e) { + // This is thrown if the BundleContext is no longer valid which is possible on shutdown so we + // log as debug. + LOG.debug("Error obtaining OSGi service {}", serviceInterface.getSimpleName(), e); + } + + return null; + } } diff --git a/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java b/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java index d0e39873d9..e70fc372b7 100644 --- a/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java +++ b/opendaylight/blueprint/src/main/java/org/opendaylight/controller/blueprint/BlueprintContainerRestartServiceImpl.java @@ -77,6 +77,10 @@ class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintCo @Override public void restartContainerAndDependents(final Bundle bundle) { + if(restartExecutor.isShutdown()) { + return; + } + LOG.debug("restartContainerAndDependents for bundle {}", bundle); restartExecutor.execute(new Runnable() { diff --git a/opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java b/opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java new file mode 100644 index 0000000000..9a1470ac3e --- /dev/null +++ b/opendaylight/config/config-api/src/main/java/org/opendaylight/controller/config/api/ConfigSystemService.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.config.api; + +/** + * Service interface for the config system. + * + * @author Thomas Pantelis + */ +public interface ConfigSystemService { + /** + * This method closes all the config system modules. This method should only be called on process + * shutdown and is provided as a hook to control the shutdown sequence. + */ + void closeAllConfigModules(); +} diff --git a/opendaylight/config/config-manager/src/main/java/org/opendaylight/controller/config/manager/impl/osgi/ConfigManagerActivator.java b/opendaylight/config/config-manager/src/main/java/org/opendaylight/controller/config/manager/impl/osgi/ConfigManagerActivator.java index ec27bc724f..781debb256 100644 --- a/opendaylight/config/config-manager/src/main/java/org/opendaylight/controller/config/manager/impl/osgi/ConfigManagerActivator.java +++ b/opendaylight/config/config-manager/src/main/java/org/opendaylight/controller/config/manager/impl/osgi/ConfigManagerActivator.java @@ -15,6 +15,7 @@ import java.util.Collection; import java.util.List; import javax.management.InstanceAlreadyExistsException; import javax.management.MBeanServer; +import org.opendaylight.controller.config.api.ConfigSystemService; import org.opendaylight.controller.config.api.ConfigRegistry; import org.opendaylight.controller.config.manager.impl.ConfigRegistryImpl; import org.opendaylight.controller.config.manager.impl.jmx.ConfigRegistryJMXRegistrator; @@ -38,7 +39,7 @@ import org.osgi.util.tracker.ServiceTracker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ConfigManagerActivator implements BundleActivator, SynchronousBundleListener { +public class ConfigManagerActivator implements BundleActivator, SynchronousBundleListener, ConfigSystemService { private static final Logger LOG = LoggerFactory.getLogger(ConfigManagerActivator.class); @@ -123,9 +124,12 @@ public class ConfigManagerActivator implements BundleActivator, SynchronousBundl blankTransactionServiceTracker); serviceTracker.open(); + AutoCloseable configMgrReg = registerService(context, this, ConfigSystemService.class); + List list = Arrays.asList(bindingContextProvider, clsReg, wrap(moduleFactoryBundleTracker), moduleInfoBundleTracker, - configRegReg, configRegistryJMXRegistrator, configRegistryJMXRegistratorWithNotifications, wrap(serviceTracker), moduleInfoRegistryWrapper, notifyingConfigRegistry); + configRegReg, configRegistryJMXRegistrator, configRegistryJMXRegistratorWithNotifications, + wrap(serviceTracker), moduleInfoRegistryWrapper, notifyingConfigRegistry, configMgrReg); autoCloseable = OsgiRegistrationUtil.aggregate(list); context.addBundleListener(this); @@ -157,4 +161,11 @@ public class ConfigManagerActivator implements BundleActivator, SynchronousBundl configRegistry.close(); } } + + @Override + public void closeAllConfigModules() { + if(configRegistry != null) { + configRegistry.close(); + } + } } -- 2.36.6