2 * Copyright (c) 2015 - 2016 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.vpnmanager;
10 import com.google.common.base.Optional;
11 import com.google.common.util.concurrent.CheckedFuture;
12 import com.google.common.util.concurrent.FutureCallback;
13 import com.google.common.util.concurrent.Futures;
14 import com.google.common.util.concurrent.ListenableFuture;
15 import com.google.common.util.concurrent.ThreadFactoryBuilder;
16 import java.util.ArrayList;
17 import java.util.List;
18 import java.util.concurrent.Callable;
19 import java.util.concurrent.ConcurrentHashMap;
20 import java.util.concurrent.ConcurrentMap;
21 import java.util.concurrent.ExecutionException;
22 import java.util.concurrent.ExecutorService;
23 import java.util.concurrent.Executors;
24 import java.util.concurrent.ThreadFactory;
25 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
26 import org.opendaylight.controller.md.sal.binding.api.DataChangeListener;
27 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
28 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker;
29 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
30 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
31 import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator;
32 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
33 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
34 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnAfConfig;
35 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
36 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
37 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.id.to.vpn.instance.VpnIds;
40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntryBuilder;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
43 import org.opendaylight.yangtools.concepts.ListenerRegistration;
44 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
48 public class VpnInstanceListener extends AbstractDataChangeListener<VpnInstance> implements AutoCloseable {
49 private static final Logger LOG = LoggerFactory.getLogger(VpnInstanceListener.class);
50 private ListenerRegistration<DataChangeListener> listenerRegistration;
51 private final DataBroker dataBroker;
52 private final IBgpManager bgpManager;
53 private final IdManagerService idManager;
54 private final VpnInterfaceManager vpnInterfaceManager;
55 private final IFibManager fibManager;
56 private static final ThreadFactory threadFactory = new ThreadFactoryBuilder()
57 .setNameFormat("NV-VpnMgr-%d").build();
58 private ExecutorService executorService = Executors.newSingleThreadExecutor(threadFactory);
59 private ConcurrentMap<String, Runnable> vpnOpMap = new ConcurrentHashMap<String, Runnable>();
61 public VpnInstanceListener(final DataBroker dataBroker, final IBgpManager bgpManager,
62 final IdManagerService idManager,
63 final VpnInterfaceManager vpnInterfaceManager,
64 final IFibManager fibManager) {
65 super(VpnInstance.class);
66 this.dataBroker = dataBroker;
67 this.bgpManager = bgpManager;
68 this.idManager = idManager;
69 this.vpnInterfaceManager = vpnInterfaceManager;
70 this.fibManager = fibManager;
74 LOG.info("{} start", getClass().getSimpleName());
75 listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION,
76 getWildCardPath(), this, AsyncDataBroker.DataChangeScope.SUBTREE);
79 private InstanceIdentifier<VpnInstance> getWildCardPath() {
80 return InstanceIdentifier.create(VpnInstances.class).child(VpnInstance.class);
84 public void close() throws Exception {
85 if (listenerRegistration != null) {
86 listenerRegistration.close();
87 listenerRegistration = null;
89 LOG.info("{} close", getClass().getSimpleName());
92 void notifyTaskIfRequired(String vpnName) {
93 Runnable notifyTask = vpnOpMap.remove(vpnName);
94 if (notifyTask == null) {
95 LOG.trace("VpnInstanceListener update: No Notify Task queued for vpnName {}", vpnName);
98 executorService.execute(notifyTask);
101 private void waitForOpRemoval(String rd, String vpnName) {
102 //wait till DCN for update on VPN Instance Op Data signals that vpn interfaces linked to this vpn instance is zero
103 //TODO(vpnteam): Entire code would need refactoring to listen only on the parent object - VPNInstance
104 VpnInstanceOpDataEntry vpnOpEntry = null;
106 Long currentIntfCount = 0L;
107 Integer retryCount = 1;
108 long timeout = VpnConstants.MIN_WAIT_TIME_IN_MILLISECONDS;
109 Optional<VpnInstanceOpDataEntry> vpnOpValue = null;
110 vpnOpValue = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
111 VpnUtil.getVpnInstanceOpDataIdentifier(rd));
113 if ((vpnOpValue != null) && (vpnOpValue.isPresent())) {
114 vpnOpEntry = vpnOpValue.get();
115 List<VpnToDpnList> dpnToVpns = vpnOpEntry.getVpnToDpnList();
116 if (dpnToVpns != null) {
117 for (VpnToDpnList dpn : dpnToVpns) {
118 if (dpn.getVpnInterfaces() != null) {
119 intfCount = intfCount + dpn.getVpnInterfaces().size();
123 //intfCount = vpnOpEntry.getVpnInterfaceCount();
126 // Minimum wait time of 5 seconds for one VPN Interface clearance (inclusive of full trace on)
127 timeout = intfCount * VpnConstants.MIN_WAIT_TIME_IN_MILLISECONDS;
128 // Maximum wait time of 90 seconds for all VPN Interfaces clearance (inclusive of full trace on)
129 if (timeout > VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS) {
130 timeout = VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS;
132 LOG.info("VPNInstance removal count of interface at {} for for rd {}, vpnname {}",
133 intfCount, rd, vpnName);
135 LOG.info("VPNInstance removal thread waiting for {} seconds for rd {}, vpnname {}",
136 (timeout / 1000), rd, vpnName);
139 Thread.sleep(timeout);
140 } catch (InterruptedException e) {
143 // Check current interface count
144 vpnOpValue = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
145 VpnUtil.getVpnInstanceOpDataIdentifier(rd));
146 if ((vpnOpValue != null) && (vpnOpValue.isPresent())) {
147 vpnOpEntry = vpnOpValue.get();
148 dpnToVpns = vpnOpEntry.getVpnToDpnList();
149 currentIntfCount = 0L;
150 if (dpnToVpns != null) {
151 for (VpnToDpnList dpn : dpnToVpns) {
152 if (dpn.getVpnInterfaces() != null) {
153 currentIntfCount = currentIntfCount + dpn.getVpnInterfaces().size();
157 if ((currentIntfCount == 0) || (currentIntfCount >= intfCount)) {
158 // Either the FibManager completed its job to cleanup all vpnInterfaces in VPN
160 // There is no progress by FibManager in removing all the interfaces even after good time!
161 // In either case, let us quit and take our chances.
162 //TODO(vpnteam): L3VPN refactoring to take care of this case.
163 if ((dpnToVpns == null) || dpnToVpns.size() <= 0) {
164 LOG.info("VPN Instance vpn {} rd {} ready for removal, exiting wait loop", vpnName, rd);
167 if (retryCount > 0) {
169 LOG.info("Retrying clearing vpn with vpnname {} rd {} since current interface count {} ", vpnName, rd, currentIntfCount);
170 if (currentIntfCount > 0) {
171 intfCount = currentIntfCount;
173 LOG.info("Current interface count is zero, but instance Op for vpn {} and rd {} not cleared yet. Waiting for 5 more seconds.", vpnName, rd);
177 LOG.info("VPNInstance bailing out of wait loop as current interface count is {} and max retries exceeded for for vpnName {}, rd {}",
178 currentIntfCount, vpnName, rd);
184 // There is no VPNOPEntry. Something else happened on the system !
185 // So let us quit and take our chances.
186 //TODO(vpnteam): L3VPN refactoring to take care of this case.
191 LOG.info("Returned out of waiting for Op Data removal for rd {}, vpnname {}", rd, vpnName);
194 protected void remove(InstanceIdentifier<VpnInstance> identifier, VpnInstance del) {
195 LOG.trace("Remove VPN event key: {}, value: {}", identifier, del);
196 final String vpnName = del.getVpnInstanceName();
197 final String rd = del.getIpv4Family().getRouteDistinguisher();
198 final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
199 Optional<VpnInstanceOpDataEntry> vpnOpValue = null;
201 //TODO(vpnteam): Entire code would need refactoring to listen only on the parent object - VPNInstance
203 if ((rd != null) && (!rd.isEmpty())) {
204 vpnOpValue = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
205 VpnUtil.getVpnInstanceOpDataIdentifier(rd));
207 vpnOpValue = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
208 VpnUtil.getVpnInstanceOpDataIdentifier(vpnName));
210 } catch (Exception e) {
211 LOG.error("Exception when attempting to retrieve VpnInstanceOpDataEntry for VPN {}. ", vpnName, e);
215 if (vpnOpValue == null || !vpnOpValue.isPresent()) {
216 LOG.error("Unable to retrieve VpnInstanceOpDataEntry for VPN {}. ", vpnName);
220 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
221 dataStoreCoordinator.enqueueJob("VPN-" + vpnName,
222 new DeleteVpnInstanceWorker(idManager, dataBroker, del));
225 private class DeleteVpnInstanceWorker implements Callable<List<ListenableFuture<Void>>> {
226 IdManagerService idManager;
228 VpnInstance vpnInstance;
230 public DeleteVpnInstanceWorker(IdManagerService idManager,
233 this.idManager = idManager;
234 this.broker = broker;
235 this.vpnInstance = value;
239 public List<ListenableFuture<Void>> call() throws Exception {
240 final String vpnName = vpnInstance.getVpnInstanceName();
241 final String rd = vpnInstance.getIpv4Family().getRouteDistinguisher();
242 final long vpnId = VpnUtil.getVpnId(broker, vpnName);
243 WriteTransaction writeTxn = broker.newWriteOnlyTransaction();
244 if ((rd != null) && (!rd.isEmpty())) {
245 waitForOpRemoval(rd, vpnName);
247 waitForOpRemoval(vpnName, vpnName);
250 // Clean up VpnInstanceToVpnId from Config DS
251 VpnUtil.removeVpnIdToVpnInstance(broker, vpnId, writeTxn);
252 VpnUtil.removeVpnInstanceToVpnId(broker, vpnName, writeTxn);
254 List<ListenableFuture<Void>> futures = new ArrayList<>();
255 futures.add(writeTxn.submit());
256 LOG.trace("Removed vpnIdentifier for rd{} vpnname {}", rd, vpnName);
258 synchronized (vpnName.intern()) {
259 fibManager.removeVrfTable(broker, rd, null);
262 bgpManager.deleteVrf(rd, false);
263 } catch (Exception e) {
264 LOG.error("Exception when removing VRF from BGP for RD {} in VPN {} exception " + e, rd, vpnName);
267 // Clean up VPNExtraRoutes Operational DS
268 VpnUtil.removeVpnExtraRouteForVpn(broker, rd, null);
270 // Clean up VPNInstanceOpDataEntry
271 VpnUtil.removeVpnOpInstance(broker, rd, null);
273 // Clean up FIB Entries Config DS
274 synchronized (vpnName.intern()) {
275 fibManager.removeVrfTable(broker, vpnName, null);
277 // Clean up VPNExtraRoutes Operational DS
278 VpnUtil.removeVpnExtraRouteForVpn(broker, vpnName, null);
280 // Clean up VPNInstanceOpDataEntry
281 VpnUtil.removeVpnOpInstance(broker, vpnName, null);
283 // Clean up PrefixToInterface Operational DS
284 VpnUtil.removePrefixToInterfaceForVpnId(broker, vpnId, null);
286 // Clean up L3NextHop Operational DS
287 VpnUtil.removeL3nexthopForVpnId(broker, vpnId, null);
289 // Release the ID used for this VPN back to IdManager
290 VpnUtil.releaseId(idManager, VpnConstants.VPN_IDPOOL_NAME, vpnName);
297 protected void update(InstanceIdentifier<VpnInstance> identifier,
298 VpnInstance original, VpnInstance update) {
299 LOG.trace("Update VPN event key: {}, value: {}", identifier, update);
303 protected void add(final InstanceIdentifier<VpnInstance> identifier, final VpnInstance value) {
304 LOG.trace("Add VPN event key: {}, value: {}", identifier, value);
305 final VpnAfConfig config = value.getIpv4Family();
306 final String rd = config.getRouteDistinguisher();
307 final String vpnName = value.getVpnInstanceName();
309 DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance();
310 dataStoreCoordinator.enqueueJob("VPN-" + vpnName,
311 new AddVpnInstanceWorker(idManager, vpnInterfaceManager, dataBroker, value));
314 private class AddVpnInstanceWorker implements Callable<List<ListenableFuture<Void>>> {
315 IdManagerService idManager;
316 VpnInterfaceManager vpnInterfaceManager;
317 VpnInstance vpnInstance;
320 public AddVpnInstanceWorker(IdManagerService idManager,
321 VpnInterfaceManager vpnInterfaceManager,
324 this.idManager = idManager;
325 this.vpnInterfaceManager = vpnInterfaceManager;
326 this.broker = broker;
327 this.vpnInstance = value;
331 public List<ListenableFuture<Void>> call() throws Exception {
332 // If another renderer(for eg : CSS) needs to be supported, check can be performed here
333 // to call the respective helpers.
334 final VpnAfConfig config = vpnInstance.getIpv4Family();
335 final String rd = config.getRouteDistinguisher();
336 WriteTransaction writeConfigTxn = broker.newWriteOnlyTransaction();
337 WriteTransaction writeOperTxn = broker.newWriteOnlyTransaction();
338 addVpnInstance(vpnInstance, writeConfigTxn, writeOperTxn);
339 CheckedFuture<Void, TransactionCommitFailedException> checkFutures = writeOperTxn.submit();
342 } catch (InterruptedException | ExecutionException e) {
343 LOG.error("Error creating vpn {} ", vpnInstance.getVpnInstanceName());
344 throw new RuntimeException(e.getMessage());
346 List<ListenableFuture<Void>> futures = new ArrayList<>();
347 futures.add(writeConfigTxn.submit());
348 ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
350 Futures.addCallback(listenableFuture,
351 new AddBgpVrfWorker(config , vpnInstance.getVpnInstanceName()));
357 private void addVpnInstance(VpnInstance value, WriteTransaction writeConfigTxn,
358 WriteTransaction writeOperTxn) {
359 VpnAfConfig config = value.getIpv4Family();
360 String rd = config.getRouteDistinguisher();
361 String vpnInstanceName = value.getVpnInstanceName();
363 long vpnId = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, vpnInstanceName);
364 LOG.trace("VPN instance to ID generated.");
365 org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.to.vpn.id.VpnInstance
366 vpnInstanceToVpnId = VpnUtil.getVpnInstanceToVpnId(vpnInstanceName, vpnId, (rd != null) ? rd
369 if (writeConfigTxn != null) {
370 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION,
371 VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnInstanceName),
372 vpnInstanceToVpnId, true);
374 TransactionUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
375 VpnUtil.getVpnInstanceToVpnIdIdentifier(vpnInstanceName),
376 vpnInstanceToVpnId, TransactionUtil.DEFAULT_CALLBACK);
379 VpnIds vpnIdToVpnInstance = VpnUtil.getVpnIdToVpnInstance(vpnId, value.getVpnInstanceName(),
380 (rd != null) ? rd : value.getVpnInstanceName(), (rd != null)/*isExternalVpn*/);
382 if (writeConfigTxn != null) {
383 writeConfigTxn.put(LogicalDatastoreType.CONFIGURATION,
384 VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId),
385 vpnIdToVpnInstance, true);
387 TransactionUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
388 VpnUtil.getVpnIdToVpnInstanceIdentifier(vpnId),
389 vpnIdToVpnInstance, TransactionUtil.DEFAULT_CALLBACK);
393 String cachedTransType = fibManager.getConfTransType();
394 LOG.trace("Value for confTransportType is " + cachedTransType);
395 if (cachedTransType.equals("Invalid")) {
397 fibManager.setConfTransType("L3VPN", "VXLAN");
398 } catch (Exception e) {
399 LOG.trace("Exception caught setting the cached value for transportType");
400 LOG.error(e.getMessage());
403 LOG.trace(":cached val is neither unset/invalid. NO-op.");
405 } catch (Exception e) {
406 LOG.error(e.getMessage());
410 VpnInstanceOpDataEntryBuilder builder =
411 new VpnInstanceOpDataEntryBuilder().setVrfId(vpnInstanceName).setVpnId(vpnId)
412 .setVpnInstanceName(vpnInstanceName)
413 .setVpnInterfaceCount(0L);
414 if (writeOperTxn != null) {
415 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
416 VpnUtil.getVpnInstanceOpDataIdentifier(vpnInstanceName),
417 builder.build(), true);
419 TransactionUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
420 VpnUtil.getVpnInstanceOpDataIdentifier(vpnInstanceName),
421 builder.build(), TransactionUtil.DEFAULT_CALLBACK);
423 synchronized (vpnInstanceName.intern()) {
424 fibManager.addVrfTable(dataBroker, vpnInstanceName, null);
427 VpnInstanceOpDataEntryBuilder builder = new VpnInstanceOpDataEntryBuilder()
428 .setVrfId(rd).setVpnId(vpnId).setVpnInstanceName(vpnInstanceName).setVpnInterfaceCount(0L);
430 if (writeOperTxn != null) {
431 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
432 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
433 builder.build(), true);
435 TransactionUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
436 VpnUtil.getVpnInstanceOpDataIdentifier(rd),
437 builder.build(), TransactionUtil.DEFAULT_CALLBACK);
439 synchronized (vpnInstanceName.intern()) {
440 fibManager.addVrfTable(dataBroker, rd, null);
446 private class AddBgpVrfWorker implements FutureCallback<List<Void>> {
450 public AddBgpVrfWorker(VpnAfConfig config, String vpnName) {
451 this.config = config;
452 this.vpnName = vpnName;
457 * This implies that all the future instances have returned success. -- TODO: Confirm this
460 public void onSuccess(List<Void> voids) {
461 String rd = config.getRouteDistinguisher();
463 List<VpnTarget> vpnTargetList = config.getVpnTargets().getVpnTarget();
465 List<String> ertList = new ArrayList<String>();
466 List<String> irtList = new ArrayList<String>();
468 for (VpnTarget vpnTarget : vpnTargetList) {
469 if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.ExportExtcommunity) {
470 ertList.add(vpnTarget.getVrfRTValue());
472 if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.ImportExtcommunity) {
473 irtList.add(vpnTarget.getVrfRTValue());
475 if (vpnTarget.getVrfRTType() == VpnTarget.VrfRTType.Both) {
476 ertList.add(vpnTarget.getVrfRTValue());
477 irtList.add(vpnTarget.getVrfRTValue());
482 bgpManager.addVrf(rd, irtList, ertList);
483 } catch (Exception e) {
484 LOG.error("Exception when adding VRF to BGP", e);
487 vpnInterfaceManager.handleVpnsExportingRoutes(this.vpnName, rd);
493 * This method is used to handle failure callbacks.
494 * If more retry needed, the retrycount is decremented and mainworker is executed again.
495 * After retries completed, rollbackworker is executed.
496 * If rollbackworker fails, this is a double-fault. Double fault is logged and ignored.
500 public void onFailure(Throwable throwable) {
501 LOG.warn("Job: failed with exception: {}", throwable.getStackTrace());
505 public boolean isVPNConfigured() {
507 InstanceIdentifier<VpnInstances> vpnsIdentifier =
508 InstanceIdentifier.builder(VpnInstances.class).build();
509 Optional<VpnInstances> optionalVpns = TransactionUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
511 if (!optionalVpns.isPresent() ||
512 optionalVpns.get().getVpnInstance() == null ||
513 optionalVpns.get().getVpnInstance().isEmpty()) {
514 LOG.trace("No VPNs configured.");
517 LOG.trace("VPNs are configured on the system.");
521 protected VpnInstanceOpDataEntry getVpnInstanceOpData(String rd) {
522 InstanceIdentifier<VpnInstanceOpDataEntry> id = VpnUtil.getVpnInstanceOpDataIdentifier(rd);
523 Optional<VpnInstanceOpDataEntry> vpnInstanceOpData =
524 TransactionUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
525 if(vpnInstanceOpData.isPresent()) {
526 return vpnInstanceOpData.get();