2 * Copyright (c) 2016 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.blueprint;
10 import com.google.common.base.Optional;
11 import com.google.common.base.Preconditions;
12 import com.google.common.collect.Lists;
13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
14 import java.lang.management.ManagementFactory;
15 import java.util.AbstractMap.SimpleEntry;
16 import java.util.ArrayDeque;
17 import java.util.ArrayList;
18 import java.util.Arrays;
19 import java.util.Collections;
20 import java.util.Deque;
21 import java.util.Dictionary;
22 import java.util.Hashtable;
23 import java.util.LinkedHashSet;
24 import java.util.List;
25 import java.util.Map.Entry;
27 import java.util.concurrent.CountDownLatch;
28 import java.util.concurrent.ExecutorService;
29 import java.util.concurrent.Executors;
30 import java.util.concurrent.TimeUnit;
31 import javax.annotation.Nullable;
32 import javax.management.InstanceNotFoundException;
33 import javax.management.ObjectName;
34 import javax.xml.parsers.ParserConfigurationException;
35 import org.apache.aries.blueprint.services.BlueprintExtenderService;
36 import org.apache.aries.quiesce.participant.QuiesceParticipant;
37 import org.apache.aries.util.AriesFrameworkUtil;
38 import org.opendaylight.controller.config.api.ConfigRegistry;
39 import org.opendaylight.controller.config.api.ConflictingVersionException;
40 import org.opendaylight.controller.config.api.ModuleIdentifier;
41 import org.opendaylight.controller.config.api.ValidationException;
42 import org.opendaylight.controller.config.facade.xml.ConfigExecution;
43 import org.opendaylight.controller.config.facade.xml.ConfigSubsystemFacade;
44 import org.opendaylight.controller.config.facade.xml.ConfigSubsystemFacadeFactory;
45 import org.opendaylight.controller.config.facade.xml.TestOption;
46 import org.opendaylight.controller.config.facade.xml.mapping.config.Config;
47 import org.opendaylight.controller.config.facade.xml.strategy.EditStrategyType;
48 import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
49 import org.opendaylight.controller.config.util.xml.DocumentedException;
50 import org.opendaylight.controller.config.util.xml.XmlElement;
51 import org.opendaylight.controller.config.util.xml.XmlMappingConstants;
52 import org.opendaylight.controller.config.util.xml.XmlUtil;
53 import org.osgi.framework.Bundle;
54 import org.osgi.framework.BundleContext;
55 import org.osgi.framework.ServiceReference;
56 import org.osgi.framework.ServiceRegistration;
57 import org.osgi.service.blueprint.container.EventConstants;
58 import org.osgi.service.event.EventHandler;
59 import org.slf4j.Logger;
60 import org.slf4j.LoggerFactory;
61 import org.w3c.dom.Document;
62 import org.w3c.dom.Element;
65 * Implementation of the BlueprintContainerRestartService.
67 * @author Thomas Pantelis
69 class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintContainerRestartService {
70 private static final Logger LOG = LoggerFactory.getLogger(BlueprintContainerRestartServiceImpl.class);
71 private static final String CONFIG_MODULE_NAMESPACE_PROP = "config-module-namespace";
72 private static final String CONFIG_MODULE_NAME_PROP = "config-module-name";
73 private static final String CONFIG_INSTANCE_NAME_PROP = "config-instance-name";
75 private final ExecutorService restartExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
76 .setDaemon(true).setNameFormat("BlueprintContainerRestartService").build());
78 private BlueprintExtenderService blueprintExtenderService;
79 private QuiesceParticipant quiesceParticipant;
81 void setBlueprintExtenderService(final BlueprintExtenderService blueprintExtenderService) {
82 this.blueprintExtenderService = blueprintExtenderService;
85 void setQuiesceParticipant(final QuiesceParticipant quiesceParticipant) {
86 this.quiesceParticipant = quiesceParticipant;
89 public void restartContainer(final Bundle bundle, final List<Object> paths) {
90 if (restartExecutor.isShutdown()) {
94 LOG.debug("restartContainer for bundle {}", bundle);
96 restartExecutor.execute(() -> {
97 blueprintExtenderService.destroyContainer(bundle, blueprintExtenderService.getContainer(bundle));
98 blueprintExtenderService.createContainer(bundle, paths);
103 public void restartContainerAndDependents(final Bundle bundle) {
104 if (restartExecutor.isShutdown()) {
108 LOG.debug("restartContainerAndDependents for bundle {}", bundle);
110 restartExecutor.execute(() -> restartContainerAndDependentsInternal(bundle));
113 private void restartContainerAndDependentsInternal(Bundle forBundle) {
114 Preconditions.checkNotNull(blueprintExtenderService);
115 Preconditions.checkNotNull(quiesceParticipant);
117 // We use a LinkedHashSet to preserve insertion order as we walk the service usage hierarchy.
118 Set<Bundle> containerBundlesSet = new LinkedHashSet<>();
119 List<Entry<String, ModuleIdentifier>> configModules = new ArrayList<>();
120 findDependentContainersRecursively(forBundle, containerBundlesSet, configModules);
122 List<Bundle> containerBundles = new ArrayList<>(containerBundlesSet);
124 LOG.info("Restarting blueprint containers for bundle {} and its dependent bundles {}", forBundle,
125 containerBundles.subList(1, containerBundles.size()));
127 // The blueprint containers are created asynchronously so we register a handler for blueprint events
128 // that are sent when a container is complete, successful or not. The CountDownLatch tells when all
129 // containers are complete. This is done to ensure all blueprint containers are finished before we
130 // restart config modules.
131 final CountDownLatch containerCreationComplete = new CountDownLatch(containerBundles.size());
132 ServiceRegistration<?> eventHandlerReg = registerEventHandler(forBundle.getBundleContext(), event -> {
133 LOG.debug("handleEvent {} for bundle {}", event.getTopic(), event.getProperty(EventConstants.BUNDLE));
134 if (containerBundles.contains(event.getProperty(EventConstants.BUNDLE))) {
135 containerCreationComplete.countDown();
139 final Runnable createContainerCallback = () -> createContainers(containerBundles);
141 // Destroy the container down-top recursively and once done, restart the container top-down
142 destroyContainers(new ArrayDeque<>(Lists.reverse(containerBundles)), createContainerCallback);
146 containerCreationComplete.await(5, TimeUnit.MINUTES);
147 } catch (InterruptedException e) {
148 LOG.debug("CountDownLatch await was interrupted - returning");
152 AriesFrameworkUtil.safeUnregisterService(eventHandlerReg);
154 // Now restart any associated config system Modules.
155 restartConfigModules(forBundle.getBundleContext(), configModules);
159 * Recursively quiesce and destroy the bundles one by one in order to maintain synchronicity and ordering.
160 * @param remainingBundlesToDestroy the list of remaining bundles to destroy.
161 * @param createContainerCallback a {@link Runnable} to {@code run()} when the recursive function is completed.
163 private void destroyContainers(final Deque<Bundle> remainingBundlesToDestroy,
164 final Runnable createContainerCallback) {
166 final Bundle nextBundle;
167 synchronized (remainingBundlesToDestroy) {
168 if (remainingBundlesToDestroy.isEmpty()) {
169 LOG.debug("All blueprint containers were quiesced and destroyed");
170 createContainerCallback.run();
174 nextBundle = remainingBundlesToDestroy.poll();
177 // The Quiesce capability is a like a soft-stop, clean-stop. In the case of the Blueprint extender, in flight
178 // service calls are allowed to finish; they're counted in and counted out, and no new calls are allowed. When
179 // there are no in flight service calls, the bundle is told to stop. The Blueprint bundle itself doesn't know
180 // this is happening which is a key design point. In the case of Blueprint, the extender ensures no new Entity
181 // Managers(EMs) are created. Then when all those EMs are closed the quiesce operation reports that it is
183 // To properly restart the blueprint containers, first we have to quiesce the list of bundles, and once done, it
184 // is safe to destroy their BlueprintContainer, so no reference is retained.
186 // Mail - thread explaining Quiesce API:
187 // https://www.mail-archive.com/dev@aries.apache.org/msg08403.html
189 // Quiesced the bundle to unregister the associated BlueprintContainer
190 quiesceParticipant.quiesce(bundlesQuiesced -> {
192 // Destroy the container once Quiesced
193 Arrays.stream(bundlesQuiesced).forEach(quiescedBundle -> {
194 LOG.debug("Quiesced bundle {}", quiescedBundle);
195 blueprintExtenderService.destroyContainer(
196 quiescedBundle, blueprintExtenderService.getContainer(quiescedBundle));
199 destroyContainers(remainingBundlesToDestroy, createContainerCallback);
201 }, Collections.singletonList(nextBundle));
204 private void createContainers(List<Bundle> containerBundles) {
205 containerBundles.forEach(bundle -> {
206 List<Object> paths = BlueprintBundleTracker.findBlueprintPaths(bundle);
208 LOG.info("Restarting blueprint container for bundle {} with paths {}", bundle, paths);
210 blueprintExtenderService.createContainer(bundle, paths);
214 private void restartConfigModules(BundleContext bundleContext, List<Entry<String,
215 ModuleIdentifier>> configModules) {
216 if (configModules.isEmpty()) {
220 ServiceReference<ConfigSubsystemFacadeFactory> configFacadeFactoryRef = bundleContext
221 .getServiceReference(ConfigSubsystemFacadeFactory.class);
222 if (configFacadeFactoryRef == null) {
223 LOG.debug("ConfigSubsystemFacadeFactory service reference not found");
227 ConfigSubsystemFacadeFactory configFacadeFactory = bundleContext.getService(configFacadeFactoryRef);
228 if (configFacadeFactory == null) {
229 LOG.debug("ConfigSubsystemFacadeFactory service not found");
233 ConfigSubsystemFacade configFacade = configFacadeFactory.createFacade("BlueprintContainerRestartService");
235 restartConfigModules(configModules, configFacade);
236 } catch (ParserConfigurationException | DocumentedException | ValidationException
237 | ConflictingVersionException e) {
238 LOG.error("Error restarting config modules", e);
240 configFacade.close();
241 bundleContext.ungetService(configFacadeFactoryRef);
246 private void restartConfigModules(List<Entry<String, ModuleIdentifier>> configModules,
247 ConfigSubsystemFacade configFacade) throws ParserConfigurationException, DocumentedException,
248 ValidationException, ConflictingVersionException {
250 Document document = XmlUtil.newDocument();
251 Element dataElement = XmlUtil.createElement(document, XmlMappingConstants.DATA_KEY, Optional.<String>absent());
252 Element modulesElement = XmlUtil.createElement(document, XmlMappingConstants.MODULES_KEY,
253 Optional.of(XmlMappingConstants.URN_OPENDAYLIGHT_PARAMS_XML_NS_YANG_CONTROLLER_CONFIG));
254 dataElement.appendChild(modulesElement);
256 Config configMapping = configFacade.getConfigMapping();
258 ConfigRegistry configRegistryClient = new ConfigRegistryJMXClient(ManagementFactory.getPlatformMBeanServer());
259 for (Entry<String, ModuleIdentifier> entry : configModules) {
260 String moduleNamespace = entry.getKey();
261 ModuleIdentifier moduleId = entry.getValue();
263 ObjectName instanceON = configRegistryClient.lookupConfigBean(moduleId.getFactoryName(),
264 moduleId.getInstanceName());
266 LOG.debug("Found config module instance ObjectName: {}", instanceON);
268 Element moduleElement = configMapping.moduleToXml(moduleNamespace, moduleId.getFactoryName(),
269 moduleId.getInstanceName(), instanceON, document);
270 modulesElement.appendChild(moduleElement);
271 } catch (InstanceNotFoundException e) {
272 LOG.warn("Error looking up config module: namespace {}, module name {}, instance {}",
273 moduleNamespace, moduleId.getFactoryName(), moduleId.getInstanceName(), e);
277 if (LOG.isDebugEnabled()) {
278 LOG.debug("Pushing config xml: {}", XmlUtil.toString(dataElement));
281 ConfigExecution execution = new ConfigExecution(configMapping, XmlElement.fromDomElement(dataElement),
282 TestOption.testThenSet, EditStrategyType.recreate);
283 configFacade.executeConfigExecution(execution);
284 configFacade.commitSilentTransaction();
288 * Recursively finds the services registered by the given bundle and the bundles using those services.
289 * User bundles that have an associated blueprint container are added to containerBundles. In addition,
290 * if a registered service has an associated config system Module, as determined via the presence of
291 * certain service properties, the ModuleIdentifier is added to the configModules list.
293 * @param bundle the bundle to traverse
294 * @param containerBundles the current set of bundles containing blueprint containers
295 * @param configModules the current set of bundles containing config modules
297 private void findDependentContainersRecursively(Bundle bundle, Set<Bundle> containerBundles,
298 List<Entry<String, ModuleIdentifier>> configModules) {
299 if (!containerBundles.add(bundle)) {
300 // Already seen this bundle...
304 ServiceReference<?>[] references = bundle.getRegisteredServices();
305 if (references != null) {
306 for (ServiceReference<?> reference : references) {
307 possiblyAddConfigModuleIdentifier(reference, configModules);
309 Bundle[] usingBundles = reference.getUsingBundles();
310 if (usingBundles != null) {
311 for (Bundle usingBundle : usingBundles) {
312 if (blueprintExtenderService.getContainer(usingBundle) != null) {
313 findDependentContainersRecursively(usingBundle, containerBundles, configModules);
321 private void possiblyAddConfigModuleIdentifier(ServiceReference<?> reference,
322 List<Entry<String, ModuleIdentifier>> configModules) {
323 Object moduleNamespace = reference.getProperty(CONFIG_MODULE_NAMESPACE_PROP);
324 if (moduleNamespace == null) {
328 String moduleName = getRequiredConfigModuleProperty(CONFIG_MODULE_NAME_PROP, moduleNamespace,
330 String instanceName = getRequiredConfigModuleProperty(CONFIG_INSTANCE_NAME_PROP, moduleNamespace,
332 if (moduleName == null || instanceName == null) {
336 LOG.debug("Found service with config module: namespace {}, module name {}, instance {}",
337 moduleNamespace, moduleName, instanceName);
339 configModules.add(new SimpleEntry<>(moduleNamespace.toString(),
340 new ModuleIdentifier(moduleName, instanceName)));
344 private String getRequiredConfigModuleProperty(String propName, Object moduleNamespace,
345 ServiceReference<?> reference) {
346 Object value = reference.getProperty(propName);
349 "OSGi service with {} property is missing property {} therefore the config module can't be restarted",
350 CONFIG_MODULE_NAMESPACE_PROP, propName);
354 return value.toString();
357 private ServiceRegistration<?> registerEventHandler(BundleContext bundleContext, EventHandler handler) {
358 Dictionary<String, Object> props = new Hashtable<>();
359 props.put(org.osgi.service.event.EventConstants.EVENT_TOPIC,
360 new String[]{EventConstants.TOPIC_CREATED, EventConstants.TOPIC_FAILURE});
361 return bundleContext.registerService(EventHandler.class.getName(), handler, props);
365 public void close() {
366 restartExecutor.shutdownNow();