05a796843aeaa9f3a6f15c31011992a1c38ad930
[controller.git] / opendaylight / blueprint / src / main / java / org / opendaylight / controller / blueprint / BlueprintContainerRestartServiceImpl.java
1 /*
2  * Copyright (c) 2016 Brocade Communications Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.blueprint;
9
10 import com.google.common.base.Optional;
11 import com.google.common.base.Preconditions;
12 import com.google.common.collect.Lists;
13 import com.google.common.util.concurrent.ThreadFactoryBuilder;
14 import java.lang.management.ManagementFactory;
15 import java.util.AbstractMap.SimpleEntry;
16 import java.util.ArrayDeque;
17 import java.util.ArrayList;
18 import java.util.Arrays;
19 import java.util.Collections;
20 import java.util.Deque;
21 import java.util.Hashtable;
22 import java.util.LinkedHashSet;
23 import java.util.List;
24 import java.util.Map.Entry;
25 import java.util.Set;
26 import java.util.concurrent.CountDownLatch;
27 import java.util.concurrent.ExecutorService;
28 import java.util.concurrent.Executors;
29 import java.util.concurrent.TimeUnit;
30 import javax.annotation.Nullable;
31 import javax.management.InstanceNotFoundException;
32 import javax.management.ObjectName;
33 import javax.xml.parsers.ParserConfigurationException;
34 import org.apache.aries.blueprint.services.BlueprintExtenderService;
35 import org.apache.aries.quiesce.participant.QuiesceParticipant;
36 import org.apache.aries.util.AriesFrameworkUtil;
37 import org.opendaylight.controller.config.api.ConfigRegistry;
38 import org.opendaylight.controller.config.api.ConflictingVersionException;
39 import org.opendaylight.controller.config.api.ModuleIdentifier;
40 import org.opendaylight.controller.config.api.ValidationException;
41 import org.opendaylight.controller.config.facade.xml.ConfigExecution;
42 import org.opendaylight.controller.config.facade.xml.ConfigSubsystemFacade;
43 import org.opendaylight.controller.config.facade.xml.ConfigSubsystemFacadeFactory;
44 import org.opendaylight.controller.config.facade.xml.TestOption;
45 import org.opendaylight.controller.config.facade.xml.mapping.config.Config;
46 import org.opendaylight.controller.config.facade.xml.strategy.EditStrategyType;
47 import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
48 import org.opendaylight.controller.config.util.xml.DocumentedException;
49 import org.opendaylight.controller.config.util.xml.XmlElement;
50 import org.opendaylight.controller.config.util.xml.XmlMappingConstants;
51 import org.opendaylight.controller.config.util.xml.XmlUtil;
52 import org.osgi.framework.Bundle;
53 import org.osgi.framework.BundleContext;
54 import org.osgi.framework.ServiceReference;
55 import org.osgi.framework.ServiceRegistration;
56 import org.osgi.service.blueprint.container.BlueprintEvent;
57 import org.osgi.service.blueprint.container.BlueprintListener;
58 import org.slf4j.Logger;
59 import org.slf4j.LoggerFactory;
60 import org.w3c.dom.Document;
61 import org.w3c.dom.Element;
62
63 /**
64  * Implementation of the BlueprintContainerRestartService.
65  *
66  * @author Thomas Pantelis
67  */
68 class BlueprintContainerRestartServiceImpl implements AutoCloseable, BlueprintContainerRestartService {
69     private static final Logger LOG = LoggerFactory.getLogger(BlueprintContainerRestartServiceImpl.class);
70     private static final int CONTAINER_CREATE_TIMEOUT_IN_MINUTES = 5;
71     private static final String CONFIG_MODULE_NAMESPACE_PROP = "config-module-namespace";
72     private static final String CONFIG_MODULE_NAME_PROP = "config-module-name";
73     private static final String CONFIG_INSTANCE_NAME_PROP = "config-instance-name";
74
75     private final ExecutorService restartExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
76             .setDaemon(true).setNameFormat("BlueprintContainerRestartService").build());
77
78     private BlueprintExtenderService blueprintExtenderService;
79     private QuiesceParticipant quiesceParticipant;
80
81     void setBlueprintExtenderService(final BlueprintExtenderService blueprintExtenderService) {
82         this.blueprintExtenderService = blueprintExtenderService;
83     }
84
85     void setQuiesceParticipant(final QuiesceParticipant quiesceParticipant) {
86         this.quiesceParticipant = quiesceParticipant;
87     }
88
89     public void restartContainer(final Bundle bundle, final List<Object> paths) {
90         LOG.debug("restartContainer for bundle {}", bundle);
91
92         if (restartExecutor.isShutdown()) {
93             LOG.debug("Already closed - returning");
94             return;
95         }
96
97         restartExecutor.execute(() -> {
98             blueprintExtenderService.destroyContainer(bundle, blueprintExtenderService.getContainer(bundle));
99             blueprintExtenderService.createContainer(bundle, paths);
100         });
101     }
102
103     @Override
104     public void restartContainerAndDependents(final Bundle bundle) {
105         if (restartExecutor.isShutdown()) {
106             return;
107         }
108
109         LOG.debug("restartContainerAndDependents for bundle {}", bundle);
110
111         restartExecutor.execute(() -> restartContainerAndDependentsInternal(bundle));
112     }
113
114     private void restartContainerAndDependentsInternal(final Bundle forBundle) {
115         Preconditions.checkNotNull(blueprintExtenderService);
116         Preconditions.checkNotNull(quiesceParticipant);
117
118         // We use a LinkedHashSet to preserve insertion order as we walk the service usage hierarchy.
119         Set<Bundle> containerBundlesSet = new LinkedHashSet<>();
120         List<Entry<String, ModuleIdentifier>> configModules = new ArrayList<>();
121         findDependentContainersRecursively(forBundle, containerBundlesSet, configModules);
122
123         List<Bundle> containerBundles = new ArrayList<>(containerBundlesSet);
124
125         LOG.info("Restarting blueprint containers for bundle {} and its dependent bundles {}", forBundle,
126                 containerBundles.subList(1, containerBundles.size()));
127
128         // The blueprint containers are created asynchronously so we register a handler for blueprint events
129         // that are sent when a container is complete, successful or not. The CountDownLatch tells when all
130         // containers are complete. This is done to ensure all blueprint containers are finished before we
131         // restart config modules.
132         final CountDownLatch containerCreationComplete = new CountDownLatch(containerBundles.size());
133         ServiceRegistration<?> eventHandlerReg = registerEventHandler(forBundle.getBundleContext(), event -> {
134             final Bundle bundle = event.getBundle();
135             if (event.isReplay()) {
136                 LOG.trace("Got replay BlueprintEvent {} for bundle {}", event.getType(), bundle);
137                 return;
138             }
139
140             LOG.debug("Got BlueprintEvent {} for bundle {}", event.getType(), bundle);
141             if (containerBundles.contains(bundle)
142                     && (event.getType() == BlueprintEvent.CREATED || event.getType() == BlueprintEvent.FAILURE)) {
143                 containerCreationComplete.countDown();
144                 LOG.debug("containerCreationComplete is now {}", containerCreationComplete.getCount());
145             }
146         });
147
148         final Runnable createContainerCallback = () -> createContainers(containerBundles);
149
150         // Destroy the container down-top recursively and once done, restart the container top-down
151         destroyContainers(new ArrayDeque<>(Lists.reverse(containerBundles)), createContainerCallback);
152
153
154         try {
155             if (!containerCreationComplete.await(CONTAINER_CREATE_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES)) {
156                 LOG.warn("Failed to restart all blueprint containers within {} minutes. Attempted to restart {} {} "
157                         + "but only {} completed restart", CONTAINER_CREATE_TIMEOUT_IN_MINUTES, containerBundles.size(),
158                         containerBundles, containerBundles.size() - containerCreationComplete.getCount());
159                 return;
160             }
161         } catch (final InterruptedException e) {
162             LOG.debug("CountDownLatch await was interrupted - returning");
163             return;
164         }
165
166         AriesFrameworkUtil.safeUnregisterService(eventHandlerReg);
167
168         // Now restart any associated config system Modules.
169         restartConfigModules(forBundle.getBundleContext(), configModules);
170
171         LOG.info("Finished restarting blueprint containers for bundle {} and its dependent bundles", forBundle);
172     }
173
174     /**
175      * Recursively quiesce and destroy the bundles one by one in order to maintain synchronicity and ordering.
176      * @param remainingBundlesToDestroy the list of remaining bundles to destroy.
177      * @param createContainerCallback a {@link Runnable} to {@code run()} when the recursive function is completed.
178      */
179     private void destroyContainers(final Deque<Bundle> remainingBundlesToDestroy,
180             final Runnable createContainerCallback) {
181
182         final Bundle nextBundle;
183         synchronized (remainingBundlesToDestroy) {
184             if (remainingBundlesToDestroy.isEmpty()) {
185                 LOG.debug("All blueprint containers were quiesced and destroyed");
186                 createContainerCallback.run();
187                 return;
188             }
189
190             nextBundle = remainingBundlesToDestroy.poll();
191         }
192
193         // The Quiesce capability is a like a soft-stop, clean-stop. In the case of the Blueprint extender, in flight
194         // service calls are allowed to finish; they're counted in and counted out, and no new calls are allowed. When
195         // there are no in flight service calls, the bundle is told to stop. The Blueprint bundle itself doesn't know
196         // this is happening which is a key design point. In the case of Blueprint, the extender ensures no new Entity
197         // Managers(EMs) are created. Then when all those EMs are closed the quiesce operation reports that it is
198         // finished.
199         // To properly restart the blueprint containers, first we have to quiesce the list of bundles, and once done, it
200         // is safe to destroy their BlueprintContainer, so no reference is retained.
201         //
202         // Mail - thread explaining Quiesce API:
203         //      https://www.mail-archive.com/dev@aries.apache.org/msg08403.html
204
205         // Quiesced the bundle to unregister the associated BlueprintContainer
206         quiesceParticipant.quiesce(bundlesQuiesced -> {
207
208             // Destroy the container once Quiesced
209             Arrays.stream(bundlesQuiesced).forEach(quiescedBundle -> {
210                 LOG.debug("Quiesced bundle {}", quiescedBundle);
211                 blueprintExtenderService.destroyContainer(
212                         quiescedBundle, blueprintExtenderService.getContainer(quiescedBundle));
213             });
214
215             destroyContainers(remainingBundlesToDestroy, createContainerCallback);
216
217         }, Collections.singletonList(nextBundle));
218     }
219
220     private void createContainers(final List<Bundle> containerBundles) {
221         containerBundles.forEach(bundle -> {
222             List<Object> paths = BlueprintBundleTracker.findBlueprintPaths(bundle);
223
224             LOG.info("Restarting blueprint container for bundle {} with paths {}", bundle, paths);
225
226             blueprintExtenderService.createContainer(bundle, paths);
227         });
228     }
229
230     private void restartConfigModules(final BundleContext bundleContext, final List<Entry<String,
231             ModuleIdentifier>> configModules) {
232         if (configModules.isEmpty()) {
233             return;
234         }
235
236         ServiceReference<ConfigSubsystemFacadeFactory> configFacadeFactoryRef = bundleContext
237                 .getServiceReference(ConfigSubsystemFacadeFactory.class);
238         if (configFacadeFactoryRef == null) {
239             LOG.debug("ConfigSubsystemFacadeFactory service reference not found");
240             return;
241         }
242
243         ConfigSubsystemFacadeFactory configFacadeFactory = bundleContext.getService(configFacadeFactoryRef);
244         if (configFacadeFactory == null) {
245             LOG.debug("ConfigSubsystemFacadeFactory service not found");
246             return;
247         }
248
249         try (ConfigSubsystemFacade configFacade = configFacadeFactory.createFacade(
250                 "BlueprintContainerRestartService")) {
251             restartConfigModules(configModules, configFacade);
252         } catch (ParserConfigurationException | DocumentedException | ValidationException
253                 | ConflictingVersionException e) {
254             LOG.error("Error restarting config modules", e);
255         } finally {
256             bundleContext.ungetService(configFacadeFactoryRef);
257         }
258
259     }
260
261     private void restartConfigModules(final List<Entry<String, ModuleIdentifier>> configModules,
262             final ConfigSubsystemFacade configFacade) throws ParserConfigurationException, DocumentedException,
263                     ValidationException, ConflictingVersionException {
264
265         Document document = XmlUtil.newDocument();
266         Element dataElement = XmlUtil.createElement(document, XmlMappingConstants.DATA_KEY, Optional.<String>absent());
267         Element modulesElement = XmlUtil.createElement(document, XmlMappingConstants.MODULES_KEY,
268                 Optional.of(XmlMappingConstants.URN_OPENDAYLIGHT_PARAMS_XML_NS_YANG_CONTROLLER_CONFIG));
269         dataElement.appendChild(modulesElement);
270
271         Config configMapping = configFacade.getConfigMapping();
272
273         ConfigRegistry configRegistryClient = new ConfigRegistryJMXClient(ManagementFactory.getPlatformMBeanServer());
274         for (Entry<String, ModuleIdentifier> entry : configModules) {
275             String moduleNamespace = entry.getKey();
276             ModuleIdentifier moduleId = entry.getValue();
277             try {
278                 ObjectName instanceON = configRegistryClient.lookupConfigBean(moduleId.getFactoryName(),
279                         moduleId.getInstanceName());
280
281                 LOG.debug("Found config module instance ObjectName: {}", instanceON);
282
283                 Element moduleElement = configMapping.moduleToXml(moduleNamespace, moduleId.getFactoryName(),
284                         moduleId.getInstanceName(), instanceON, document);
285                 modulesElement.appendChild(moduleElement);
286             } catch (final InstanceNotFoundException e) {
287                 LOG.warn("Error looking up config module: namespace {}, module name {}, instance {}",
288                         moduleNamespace, moduleId.getFactoryName(), moduleId.getInstanceName(), e);
289             }
290         }
291
292         if (LOG.isDebugEnabled()) {
293             LOG.debug("Pushing config xml: {}", XmlUtil.toString(dataElement));
294         }
295
296         ConfigExecution execution = new ConfigExecution(configMapping, XmlElement.fromDomElement(dataElement),
297                 TestOption.testThenSet, EditStrategyType.recreate);
298         configFacade.executeConfigExecution(execution);
299         configFacade.commitSilentTransaction();
300     }
301
302     /**
303      * Recursively finds the services registered by the given bundle and the bundles using those services.
304      * User bundles that have an associated blueprint container are added to containerBundles. In addition,
305      * if a registered service has an associated config system Module, as determined via the presence of
306      * certain service properties, the ModuleIdentifier is added to the configModules list.
307      *
308      * @param bundle the bundle to traverse
309      * @param containerBundles the current set of bundles containing blueprint containers
310      * @param configModules the current set of bundles containing config modules
311      */
312     private void findDependentContainersRecursively(final Bundle bundle, final Set<Bundle> containerBundles,
313             final List<Entry<String, ModuleIdentifier>> configModules) {
314         if (!containerBundles.add(bundle)) {
315             // Already seen this bundle...
316             return;
317         }
318
319         ServiceReference<?>[] references = bundle.getRegisteredServices();
320         if (references != null) {
321             for (ServiceReference<?> reference : references) {
322                 possiblyAddConfigModuleIdentifier(reference, configModules);
323
324                 Bundle[] usingBundles = reference.getUsingBundles();
325                 if (usingBundles != null) {
326                     for (Bundle usingBundle : usingBundles) {
327                         if (blueprintExtenderService.getContainer(usingBundle) != null) {
328                             findDependentContainersRecursively(usingBundle, containerBundles, configModules);
329                         }
330                     }
331                 }
332             }
333         }
334     }
335
336     private void possiblyAddConfigModuleIdentifier(final ServiceReference<?> reference,
337             final List<Entry<String, ModuleIdentifier>> configModules) {
338         Object moduleNamespace = reference.getProperty(CONFIG_MODULE_NAMESPACE_PROP);
339         if (moduleNamespace == null) {
340             return;
341         }
342
343         String moduleName = getRequiredConfigModuleProperty(CONFIG_MODULE_NAME_PROP, moduleNamespace,
344                 reference);
345         String instanceName = getRequiredConfigModuleProperty(CONFIG_INSTANCE_NAME_PROP, moduleNamespace,
346                 reference);
347         if (moduleName == null || instanceName == null) {
348             return;
349         }
350
351         LOG.debug("Found service with config module: namespace {}, module name {}, instance {}",
352                 moduleNamespace, moduleName, instanceName);
353
354         configModules.add(new SimpleEntry<>(moduleNamespace.toString(),
355                 new ModuleIdentifier(moduleName, instanceName)));
356     }
357
358     @Nullable
359     private String getRequiredConfigModuleProperty(final String propName, final Object moduleNamespace,
360             final ServiceReference<?> reference) {
361         Object value = reference.getProperty(propName);
362         if (value == null) {
363             LOG.warn(
364                 "OSGi service with {} property is missing property {} therefore the config module can't be restarted",
365                 CONFIG_MODULE_NAMESPACE_PROP, propName);
366             return null;
367         }
368
369         return value.toString();
370     }
371
372     private ServiceRegistration<?> registerEventHandler(final BundleContext bundleContext,
373             final BlueprintListener listener) {
374         return bundleContext.registerService(BlueprintListener.class.getName(), listener, new Hashtable<>());
375     }
376
377     @Override
378     public void close() {
379         LOG.debug("Closing");
380
381         restartExecutor.shutdownNow();
382     }
383 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.