2 * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.vpnmanager;
10 import com.google.common.base.Optional;
11 import com.google.common.base.Preconditions;
12 import com.google.common.collect.Iterators;
13 import com.google.common.util.concurrent.FutureCallback;
14 import com.google.common.util.concurrent.Futures;
15 import com.google.common.util.concurrent.ListenableFuture;
16 import com.google.common.util.concurrent.MoreExecutors;
17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
18 import java.math.BigInteger;
19 import java.util.ArrayList;
20 import java.util.Collections;
21 import java.util.Iterator;
22 import java.util.List;
24 import java.util.Objects;
25 import java.util.concurrent.ConcurrentHashMap;
26 import java.util.concurrent.ConcurrentLinkedQueue;
27 import java.util.function.Consumer;
28 import java.util.function.Predicate;
29 import java.util.stream.Collectors;
30 import javax.annotation.PostConstruct;
31 import javax.annotation.PreDestroy;
32 import javax.inject.Inject;
33 import javax.inject.Singleton;
34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
44 import org.opendaylight.genius.mdsalutil.NWUtil;
45 import org.opendaylight.genius.mdsalutil.NwConstants;
46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
48 import org.opendaylight.infrautils.caches.CacheProvider;
49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
108 import org.slf4j.Logger;
109 import org.slf4j.LoggerFactory;
112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
114 private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
115 private static final short DJC_MAX_RETRIES = 3;
117 private final DataBroker dataBroker;
118 private final ManagedNewTransactionRunner txRunner;
119 private final IBgpManager bgpManager;
120 private final IFibManager fibManager;
121 private final IMdsalApiManager mdsalManager;
122 private final IdManagerService idManager;
123 private final OdlInterfaceRpcService ifaceMgrRpcService;
124 private final VpnFootprintService vpnFootprintService;
125 private final IInterfaceManager interfaceManager;
126 private final IVpnManager vpnManager;
127 private final ArpResponderHandler arpResponderHandler;
128 private final JobCoordinator jobCoordinator;
129 private final VpnUtil vpnUtil;
131 private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
133 private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
134 new ConcurrentHashMap<>();
136 private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
139 public VpnInterfaceManager(final DataBroker dataBroker,
140 final IBgpManager bgpManager,
141 final IdManagerService idManager,
142 final IMdsalApiManager mdsalManager,
143 final IFibManager fibManager,
144 final OdlInterfaceRpcService ifaceMgrRpcService,
145 final VpnFootprintService vpnFootprintService,
146 final IInterfaceManager interfaceManager,
147 final IVpnManager vpnManager,
148 final ArpResponderHandler arpResponderHandler,
149 final JobCoordinator jobCoordinator,
150 final CacheProvider cacheProvider,
151 final VpnUtil vpnUtil) {
152 super(VpnInterface.class, VpnInterfaceManager.class);
154 this.dataBroker = dataBroker;
155 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
156 this.bgpManager = bgpManager;
157 this.idManager = idManager;
158 this.mdsalManager = mdsalManager;
159 this.fibManager = fibManager;
160 this.ifaceMgrRpcService = ifaceMgrRpcService;
161 this.vpnFootprintService = vpnFootprintService;
162 this.interfaceManager = interfaceManager;
163 this.vpnManager = vpnManager;
164 this.arpResponderHandler = arpResponderHandler;
165 this.jobCoordinator = jobCoordinator;
166 this.vpnUtil = vpnUtil;
168 vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
169 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
170 VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
173 public Runnable isNotifyTaskQueued(String intfName) {
174 return vpnIntfMap.remove(intfName);
178 public void start() {
179 LOG.info("{} start", getClass().getSimpleName());
180 registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
185 public void close() {
187 vpnInstanceOpDataEntryCache.close();
191 protected InstanceIdentifier<VpnInterface> getWildCardPath() {
192 return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
196 protected VpnInterfaceManager getDataTreeChangeListener() {
197 return VpnInterfaceManager.this;
201 public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
202 LOG.info("add: intfName {} onto vpnName {}",
203 vpnInterface.getName(),
204 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
205 addVpnInterface(identifier, vpnInterface, null, null);
208 private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
209 final VpnInterface vpnInterface, String vpnName) {
210 synchronized (vpnName.intern()) {
211 if (isVpnInstanceReady(vpnName)) {
214 addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
219 // TODO Clean up the exception handling
220 @SuppressWarnings("checkstyle:IllegalCatch")
221 private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
222 final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
223 for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
224 String vpnName = vpnInterfaceVpnInstance.getVpnName();
225 addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
229 private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
230 final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
231 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
232 final String interfaceName = key.getName();
234 if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
235 LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
236 vpnName, vpnInterface.getName());
239 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
240 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
241 List<Adjacency> copyOldAdjs = null;
242 if (oldAdjs != null) {
243 copyOldAdjs = new ArrayList<>();
244 copyOldAdjs.addAll(oldAdjs);
246 List<Adjacency> copyNewAdjs = null;
247 if (newAdjs != null) {
248 copyNewAdjs = new ArrayList<>();
249 copyNewAdjs.addAll(newAdjs);
251 addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
254 private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
255 final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
256 final List<Adjacency> newAdjs,
257 final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
258 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
259 final String interfaceName = key.getName();
260 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
261 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
262 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
263 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
264 if (interfaceState != null) {
266 final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
267 final int ifIndex = interfaceState.getIfIndex();
268 jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
269 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
270 // (the inventory tx goes in last)
271 List<ListenableFuture<Void>> futures = new ArrayList<>();
272 ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
273 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
274 operTx -> futures.add(
275 txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
277 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
279 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
280 processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
281 confTx, operTx, invTx, interfaceState, vpnName);
282 if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
283 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
284 + " Update for swapping VPN {} case.", interfaceName, vpnName);
285 if (newAdjs != null) {
286 for (Adjacency adj : newAdjs) {
287 if (oldAdjs.contains(adj)) {
290 if (!isBgpVpnInternetVpn
291 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
292 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
293 primaryRd, adj, dpnId, operTx, confTx);
298 for (Adjacency adj : oldAdjs) {
299 if (!isBgpVpnInternetVpn
300 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
301 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
307 futures.add(confFuture);
308 Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
309 MoreExecutors.directExecutor());
310 LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
311 + " processed successfully", interfaceName, vpnName, dpnId);
314 } catch (NumberFormatException | IllegalStateException e) {
315 LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
316 + "interface {}. Interface addition on vpn {} failed", interfaceName,
320 } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
321 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
323 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
324 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
326 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
327 vpnName, vpnInterface.getDpnId());
329 ListenableFutures.addErrorLogging(future, LOG,
330 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
331 return Collections.singletonList(future);
334 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
335 + " is not available", interfaceName, vpnName);
338 LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
339 + " as vpn is pending delete", interfaceName, vpnName,
340 vpnInterface.getDpnId());
344 // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
345 // see comments below.
346 @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
347 protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
348 final int lportTag, boolean isInterfaceUp,
349 WriteTransaction writeConfigTxn,
350 WriteTransaction writeOperTxn,
351 WriteTransaction writeInvTxn,
352 Interface interfaceState,
353 final String vpnName) {
354 final String interfaceName = vpnInterface.getName();
355 Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
357 VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
358 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
359 if (!isInterfaceUp) {
360 LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
361 interfaceName, dpId, vpnName);
362 long vpnId = vpnUtil.getVpnId(vpnName);
363 if (vpnId == VpnConstants.INVALID_ID) {
364 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
365 + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
370 boolean waitForVpnInterfaceOpRemoval = false;
371 if (opVpnInterface != null) {
372 String opVpnName = opVpnInterface.getVpnInstanceName();
373 String primaryInterfaceIp = null;
374 if (opVpnName.equals(vpnName)) {
375 // Please check if the primary VRF Entry does not exist for VPNInterface
376 // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
378 // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
379 List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
381 LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
382 + " for this vpn interface could not be obtained", interfaceName, dpId,
386 for (Adjacency adj : adjs) {
387 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
388 primaryInterfaceIp = adj.getIpAddress();
392 if (primaryInterfaceIp == null) {
393 LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
394 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
398 // Get the rd of the vpn instance
399 VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
401 LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
402 + " bailing out from here.", interfaceName, dpId, vpnName);
405 waitForVpnInterfaceOpRemoval = true;
407 LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
408 + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
411 if (!waitForVpnInterfaceOpRemoval) {
412 // Add the VPNInterface and quit
413 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
414 null/*ipAddressSourceValuePair*/,
416 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
417 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
418 if (!isBgpVpnInternetVpn) {
419 vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
421 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
423 if (interfaceManager.isExternalInterface(interfaceName)) {
424 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
425 NwConstants.ADD_FLOW);
430 // FIB didn't get a chance yet to clean up this VPNInterface
431 // Let us give it a chance here !
432 LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
433 + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
435 Runnable notifyTask = new VpnNotifyTask();
436 synchronized (notifyTask) {
437 // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
438 // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
439 // published here while holding the lock on it so this path will hit the wait before notify can be
441 vpnIntfMap.put(interfaceName, notifyTask);
443 notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
444 } catch (InterruptedException e) {
449 vpnIntfMap.remove(interfaceName);
452 if (opVpnInterface != null) {
453 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
454 + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
456 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
459 // VPNInterface got removed, proceed with Add
460 LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
461 interfaceName, dpId, vpnName);
462 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
463 null/*ipAddressSourceValuePair*/,
465 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
466 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
467 if (!isBgpVpnInternetVpn) {
468 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
470 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
471 + " FIB to clean up", interfaceName, dpId, vpnName);
472 if (interfaceManager.isExternalInterface(interfaceName)) {
473 processExternalVpnInterface(interfaceName, vpnName, dpId,
474 lportTag, NwConstants.ADD_FLOW);
478 // Interface is retained in the DPN, but its Link Up.
479 // Advertise prefixes again for this interface to BGP
480 InstanceIdentifier<VpnInterface> identifier =
481 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
482 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
483 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
484 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
485 // Perform similar operation as interface add event for extraroutes.
486 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
487 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
488 LogicalDatastoreType.CONFIGURATION, path);
489 if (!optAdjacencies.isPresent()) {
490 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
493 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
494 for (Adjacency adjacency : adjacencies) {
495 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
498 // if BGPVPN Internet, filter only IPv6 Adjacencies
499 if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
502 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
503 dpId, writeOperTxn, writeConfigTxn);
505 } catch (ReadFailedException e) {
506 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
507 interfaceName, vpnName, primaryRd, dpId);
512 private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
513 int lportTag, int addOrRemove) {
516 // vpn instance of ext-net interface is the network-id
517 extNetworkId = new Uuid(vpnName);
518 } catch (IllegalArgumentException e) {
519 LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
520 + " on dpn {} failed", vpnName, interfaceName, dpId);
524 List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
525 if (routerIds == null || routerIds.isEmpty()) {
526 LOG.info("processExternalVpnInterface: No router is associated with {}."
527 + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
528 extNetworkId.getValue(), interfaceName, dpId, vpnName);
532 LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
533 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
534 for (Uuid routerId : routerIds) {
535 String routerName = routerId.getValue();
536 BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
537 if (Objects.equals(primarySwitch, dpId)) {
538 Routers router = vpnUtil.getExternalRouter(routerName);
539 if (router != null) {
540 if (addOrRemove == NwConstants.ADD_FLOW) {
541 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
542 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
543 dpId, interfaceName, lportTag);
545 vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
546 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
547 dpId, interfaceName, lportTag);
550 LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
551 + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
552 interfaceName, dpId, vpnName);
558 // TODO Clean up the exception handling
559 @SuppressWarnings("checkstyle:IllegalCatch")
560 private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
561 final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
562 String vpnName, String interfaceName) {
564 LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
565 interfaceName, dpnId, vpnName);
568 if (rd.equals(vpnName)) {
569 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
570 + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
574 LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
575 interfaceName, dpnId, vpnName, rd);
577 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
578 if (nextHopIp == null) {
579 LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
580 + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
587 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
588 Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
589 LogicalDatastoreType.OPERATIONAL, path);
590 if (adjacencies.isPresent()) {
591 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
592 if (!nextHops.isEmpty()) {
593 LOG.debug("advertiseAdjacenciesForVpnToBgp: NextHops are {} for interface {} on dpn {} for vpn {}"
594 + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
595 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
596 long l3vni = vpnInstanceOpData.getL3vni();
597 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
598 ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
599 for (Adjacency nextHop : nextHops) {
600 if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
603 String gatewayMac = null;
605 if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
606 final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
607 vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
608 gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
611 label = nextHop.getLabel();
614 LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
615 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
616 bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
617 encapType, (int)label, l3vni, 0 /*l2vni*/,
619 LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
620 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
621 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
622 } catch (Exception e) {
623 LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
624 + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
625 interfaceName, dpnId, e);
630 } catch (ReadFailedException e) {
631 LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
632 + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
636 // TODO Clean up the exception handling
637 @SuppressWarnings("checkstyle:IllegalCatch")
638 private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
639 String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
641 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
642 String rd = vpnUtil.getVpnRd(interfaceName);
644 LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
645 interfaceName, vpnName);
648 if (rd.equals(vpnName)) {
650 "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
651 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
655 LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
657 Optional<AdjacenciesOp> adjacencies = Optional.absent();
659 adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
661 } catch (ReadFailedException e) {
662 LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
663 interfaceName, vpnName);
665 if (adjacencies.isPresent()) {
666 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
668 if (!nextHops.isEmpty()) {
669 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
670 nextHops, interfaceName, vpnName, rd);
671 for (Adjacency nextHop : nextHops) {
673 if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
674 LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
675 + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
676 interfaceName, vpnName);
677 bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
678 LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
679 + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
680 interfaceName, vpnName);
682 // Perform similar operation as interface delete event for extraroutes.
683 String allocatedRd = nextHop.getVrfId();
684 for (String nh : nextHop.getNextHopIpList()) {
685 deleteExtraRouteFromCurrentAndImportingVpns(
686 vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
689 } catch (Exception e) {
690 LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
691 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
698 @SuppressWarnings("checkstyle:IllegalCatch")
699 protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
700 String primaryRd, String interfaceName, final long vpnId,
701 WriteTransaction writeConfigTxn,
702 WriteTransaction writeOperTxn,
703 final WriteTransaction writeInvTxn,
704 Interface interfaceState) {
705 InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
707 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
708 Optional<Adjacencies> adjacencies = Optional.absent();
710 adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
712 } catch (ReadFailedException e) {
713 LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
714 + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
716 if (!adjacencies.isPresent()) {
717 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
718 null/*gwMac*/, writeOperTxn);
721 // Get the rd of the vpn instance
722 String nextHopIp = null;
724 nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
725 } catch (Exception e) {
726 LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
727 + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
729 List<String> nhList = new ArrayList<>();
730 if (nextHopIp != null) {
731 nhList.add(nextHopIp);
732 LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
733 interfaceName, dpnId, vpnName, nhList);
735 Optional<String> gwMac = Optional.absent();
736 String vpnInterfaceSubnetGwMacAddress = null;
737 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
738 Long l3vni = vpnInstanceOpData.getL3vni();
739 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
740 VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
741 VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
742 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
743 List<Adjacency> value = new ArrayList<>();
744 for (Adjacency nextHop : nextHops) {
745 String rd = primaryRd;
746 String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
747 if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
748 && NWUtil.isIpv4Address(nexthopIpValue)) {
749 String prefix = nextHop.getIpAddress() == null ? "null" :
750 VpnUtil.getIpPrefix(nextHop.getIpAddress());
751 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
752 + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
755 if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
756 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
757 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
758 ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
759 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
760 + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
762 LogicalDatastoreType.OPERATIONAL,
763 VpnUtil.getPrefixToInterfaceIdentifier(
764 vpnUtil.getVpnId(vpnName), prefix),
765 VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
767 final Uuid subnetId = nextHop.getSubnetId();
769 String gatewayIp = nextHop.getSubnetGatewayIp();
770 if (gatewayIp == null) {
771 Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
772 if (gatewayIpOptional.isPresent()) {
773 gatewayIp = gatewayIpOptional.get();
777 if (gatewayIp != null) {
778 gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
779 if (gwMac.isPresent()) {
780 // A valid mac-address is available for this subnet-gateway-ip
781 // Use this for programming ARP_RESPONDER table here. And save this
782 // info into vpnInterface operational, so it can used in VrfEntryProcessor
783 // to populate L3_GW_MAC_TABLE there.
784 arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
785 gatewayIp, gwMac.get());
786 vpnInterfaceSubnetGwMacAddress = gwMac.get();
788 // A valid mac-address is not available for this subnet-gateway-ip
789 // Use the connected-mac-address to configure ARP_RESPONDER Table.
790 // Save this connected-mac-address as gateway-mac-address for the
791 // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
792 gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
793 if (gwMac.isPresent()) {
794 vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
795 NwConstants.ADD_FLOW, gwMac.get());
796 arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
797 gatewayIp, gwMac.get());
799 LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
800 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
802 subnetId, interfaceName, vpnName, gatewayIp);
806 LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
807 + "cannot create ARP responder flow for interface name {}, vpnName {}",
808 subnetId, interfaceName, vpnName);
809 gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
811 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
812 + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
814 //Extra route adjacency
815 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
816 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
817 synchronized (vpnPrefixKey.intern()) {
818 java.util.Optional<String> rdToAllocate = vpnUtil
819 .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
820 nextHop.getNextHopIpList().get(0), dpnId);
821 if (rdToAllocate.isPresent()) {
822 rd = rdToAllocate.get();
823 LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
826 LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
830 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
831 + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
832 interfaceName, dpnId);
834 // Please note that primary adjacency will use a subnet-gateway-mac-address that
835 // can be different from the gateway-mac-address within the VRFEntry as the
836 // gateway-mac-address is a superset.
837 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
838 : RouteOrigin.STATIC;
839 L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
840 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
841 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
842 Adjacency operationalAdjacency = null;
844 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
845 } catch (NullPointerException e) {
846 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
847 input, e.getMessage());
850 if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
851 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
852 vpnName, l3vni, origin,
853 interfaceName, operationalAdjacency, encapType, writeConfigTxn);
855 value.add(operationalAdjacency);
858 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
859 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
860 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
862 L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
863 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
864 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
866 for (Adjacency nextHop : aug.getAdjacency()) {
867 // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
868 if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
869 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
870 : RouteOrigin.STATIC;
871 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
872 registeredPopulator.populateFib(input, writeConfigTxn);
877 private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
878 long lportTag, String gwMac, WriteTransaction writeOperTxn) {
879 VpnInterfaceOpDataEntry opInterface =
880 VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
881 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
882 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
883 writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
884 WriteTransaction.CREATE_MISSING_PARENTS);
885 LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
886 interfaceName, dpnId, vpnName);
889 // TODO Clean up the exception handling
890 @SuppressWarnings("checkstyle:IllegalCatch")
891 public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
892 StateTunnelList stateTunnelList,
893 WriteTransaction writeConfigTxn,
894 WriteTransaction writeOperTxn) {
896 String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
897 BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
898 AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
899 List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
900 if (adjList.isEmpty()) {
901 LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
902 vpnInterface, srcDpnId);
905 String prefix = null;
907 List<Adjacency> value = new ArrayList<>();
908 boolean isNextHopAddReqd = false;
909 String vpnName = vpnInterface.getVpnInstanceName();
910 long vpnId = vpnUtil.getVpnId(vpnName);
911 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
912 LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
913 vpnInterface.getName(), vpnInterface.getDpnId(),
914 vpnInterface.getVpnInstanceName(), adjList);
915 for (Adjacency adj : adjList) {
916 String rd = adj.getVrfId();
917 rd = rd != null ? rd : vpnName;
918 prefix = adj.getIpAddress();
919 label = adj.getLabel();
920 List<String> nhList = Collections.singletonList(srcTepIp);
921 List<String> nextHopList = adj.getNextHopIpList();
922 // If TEP is added , update the nexthop of primary adjacency.
923 // Secondary adj nexthop is already pointing to primary adj IP address.
924 if (nextHopList != null && !nextHopList.isEmpty()) {
925 /* everything right already */
927 isNextHopAddReqd = true;
930 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
931 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
933 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
934 if (!vrfEntryOptional.isPresent()) {
937 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
938 if (!nhList.contains(srcTepIp)) {
939 nhList.add(srcTepIp);
940 isNextHopAddReqd = true;
945 if (isNextHopAddReqd) {
946 updateLabelMapper(label, nhList);
947 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
948 + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
949 vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
950 // Update the VRF entry with nextHop
951 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
952 label, true, writeConfigTxn);
954 //Get the list of VPN's importing this route(prefix) .
955 // Then update the VRF entry with nhList
956 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
957 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
958 String vpnRd = vpn.getVrfId();
960 fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
961 srcTepIp, label, true, writeConfigTxn);
962 LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
963 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
964 vpnInterface.getName(), srcDpnId, vpnName,
965 vpn.getVpnInstanceName(), vpnRd);
968 // Advertise the prefix to BGP only for external vpn
969 // since there is a nexthop change.
971 if (!rd.equalsIgnoreCase(vpnName)) {
972 bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
973 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
974 null /*gatewayMacAddress*/);
976 LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
977 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
979 } catch (Exception ex) {
980 LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
981 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
982 vpnInterface.getName(), srcDpnId, vpnName, ex);
986 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
987 VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
988 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
989 .addAugmentation(AdjacenciesOp.class, aug).build();
990 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
991 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
992 writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
993 WriteTransaction.CREATE_MISSING_PARENTS);
994 LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
995 vpnInterface.getName(), srcDpnId, vpnName);
999 // TODO Clean up the exception handling
1000 @SuppressWarnings("checkstyle:IllegalCatch")
1001 public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1002 StateTunnelList stateTunnelList,
1003 WriteTransaction writeConfigTxn,
1004 WriteTransaction writeOperTxn) {
1006 AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1007 List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1008 String prefix = null;
1010 boolean isNextHopRemoveReqd = false;
1011 String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1012 BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1013 String vpnName = vpnInterface.getVpnInstanceName();
1014 long vpnId = vpnUtil.getVpnId(vpnName);
1015 String primaryRd = vpnUtil.getVpnRd(vpnName);
1016 if (adjList != null) {
1017 List<Adjacency> value = new ArrayList<>();
1018 LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1019 vpnInterface.getName(), vpnInterface.getDpnId(),
1020 vpnInterface.getVpnInstanceName(), adjList);
1021 for (Adjacency adj : adjList) {
1022 List<String> nhList = new ArrayList<>();
1023 String rd = adj.getVrfId();
1024 rd = rd != null ? rd : vpnName;
1025 prefix = adj.getIpAddress();
1026 List<String> nextHopList = adj.getNextHopIpList();
1027 label = adj.getLabel();
1028 if (nextHopList != null && !nextHopList.isEmpty()) {
1029 isNextHopRemoveReqd = true;
1031 // If TEP is deleted , remove the nexthop from primary adjacency.
1032 // Secondary adj nexthop will continue to point to primary adj IP address.
1033 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1034 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1036 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1037 if (!vrfEntryOptional.isPresent()) {
1040 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1041 if (nhList.contains(srcTepIp)) {
1042 nhList.remove(srcTepIp);
1043 isNextHopRemoveReqd = true;
1048 if (isNextHopRemoveReqd) {
1049 updateLabelMapper(label, nhList);
1050 LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1051 + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1052 prefix, nhList, vpnName,
1053 vpnId, rd, vpnInterface.getName());
1054 // Update the VRF entry with removed nextHop
1055 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1056 label, false, writeConfigTxn);
1058 //Get the list of VPN's importing this route(prefix) .
1059 // Then update the VRF entry with nhList
1060 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1061 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1062 String vpnRd = vpn.getVrfId();
1063 if (vpnRd != null) {
1064 fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1065 srcTepIp, label, false, writeConfigTxn);
1066 LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1067 + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1068 nhList, label, vpnInterface.getName(), srcDpnId,
1070 vpn.getVpnInstanceName(), vpnRd);
1074 // Withdraw prefix from BGP only for external vpn.
1076 if (!rd.equalsIgnoreCase(vpnName)) {
1077 bgpManager.withdrawPrefix(rd, prefix);
1079 LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1080 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1081 vpnInterface.getName(), srcDpnId,
1083 } catch (Exception ex) {
1084 LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1085 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1086 vpnInterface.getName(), srcDpnId, vpnName, ex);
1090 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1091 VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1092 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1093 .addAugmentation(AdjacenciesOp.class, aug).build();
1094 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1095 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1096 writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1097 WriteTransaction.CREATE_MISSING_PARENTS);
1098 LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1099 vpnInterface.getName(), srcDpnId, vpnName);
1103 private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1104 List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1106 String vpnRd = vpnUtil.getVpnRd(vpnName);
1107 final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1108 if (vpnInstanceOpDataEntry == null) {
1109 LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1110 + " to check for vpns exporting the routes", vpnName);
1111 return vpnsToExportRoute;
1114 Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1115 if (input.getVpnInstanceName() == null) {
1116 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {} without a name",
1120 return !input.getVpnInstanceName().equals(vpnName);
1123 Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1124 Iterable<String> commonRTs =
1125 VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1126 VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1127 return Iterators.size(commonRTs.iterator()) > 0;
1131 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1132 Collectors.toList());
1133 return vpnsToExportRoute;
1136 // TODO Clean up the exception handling
1137 @SuppressWarnings("checkstyle:IllegalCatch")
1138 void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1139 List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1140 for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1141 List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1142 if (vrfEntries != null) {
1143 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1144 for (VrfEntry vrfEntry : vrfEntries) {
1146 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1147 RouteOrigin.value(vrfEntry.getOrigin()))) {
1148 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1149 + " is not a controller managed non intervpn link route. Ignoring.",
1150 vpn.getVrfId(), vrfEntry.getDestPrefix());
1153 String prefix = vrfEntry.getDestPrefix();
1154 String gwMac = vrfEntry.getGatewayMacAddress();
1155 vrfEntry.getRoutePaths().forEach(routePath -> {
1156 String nh = routePath.getNexthopAddress();
1157 int label = routePath.getLabel().intValue();
1158 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1159 vrfEntry.getOrigin()))) {
1160 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1161 + " nexthop {} label {} to vpn {} vpnRd {}",
1162 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1163 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1164 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1165 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1168 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1169 + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1170 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1171 SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1172 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1176 } catch (RuntimeException e) {
1177 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1178 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1179 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1182 }), LOG, "Error handing VPN exporting routes");
1184 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1185 vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1191 public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1192 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1193 final String interfaceName = key.getName();
1194 for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1195 String vpnName = vpnInterfaceVpnInstance.getVpnName();
1196 removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1200 private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1201 final VpnInterface vpnInterface, final String vpnName,
1202 final String interfaceName) {
1203 if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1204 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1205 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1206 deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1207 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1209 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1210 vpnInterface.getName(), vpnName);
1211 return Collections.singletonList(future);
1212 }, DJC_MAX_RETRIES);
1214 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1215 removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1219 @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1220 private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1221 final VpnInterface vpnInterface, final String vpnName,
1222 final String interfaceName, final Interface interfaceState) {
1223 LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1224 vpnName, vpnInterface.getDpnId());
1225 removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1226 jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1228 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1229 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1230 writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1231 writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1232 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1233 interfaceName, vpnName);
1234 BigInteger dpId = BigInteger.ZERO;
1236 String gwMacAddress = null;
1237 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1238 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1239 Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1241 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1242 LogicalDatastoreType.OPERATIONAL, interfaceId);
1243 } catch (ReadFailedException e) {
1244 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1248 if (interfaceState != null) {
1250 dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1251 } catch (NumberFormatException | IllegalStateException e) {
1252 LOG.error("remove: Unable to retrieve dpnId from interface operational"
1253 + " data store for interface {} on dpn {} for vpn {} Fetching"
1254 + " from vpn interface op data store. ", interfaceName,
1255 vpnInterface.getDpnId(), vpnName, e);
1256 dpId = BigInteger.ZERO;
1258 ifIndex = interfaceState.getIfIndex();
1259 gwMacAddress = interfaceState.getPhysAddress().getValue();
1261 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1262 + " from vpn interface op.", interfaceName);
1263 if (optVpnInterface.isPresent()) {
1264 VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1265 dpId = vpnOpInterface.getDpnId();
1266 ifIndex = vpnOpInterface.getLportTag().intValue();
1267 gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1269 LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1270 + " as interfaceState and vpn interface op is not"
1271 + " available", interfaceName, vpnName);
1275 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1276 optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1277 writeConfigTxn, writeOperTxn, writeInvTxn);
1279 "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1281 interfaceName, vpnInterface.getDpnId(), vpnName);
1283 futures.add(configFuture);
1284 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1286 }, DJC_MAX_RETRIES);
1289 protected void processVpnInterfaceDown(BigInteger dpId,
1290 String interfaceName,
1293 VpnInterfaceOpDataEntry vpnOpInterface,
1294 boolean isInterfaceStateDown,
1295 WriteTransaction writeConfigTxn,
1296 WriteTransaction writeOperTxn,
1297 WriteTransaction writeInvTxn) {
1298 if (vpnOpInterface == null) {
1299 LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1300 + " as it is not available in operational data store", interfaceName, dpId);
1303 final String vpnName = vpnOpInterface.getVpnInstanceName();
1304 InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1305 interfaceName, vpnName);
1306 if (!isInterfaceStateDown) {
1307 final long vpnId = vpnUtil.getVpnId(vpnName);
1308 vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName, null);
1309 final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1310 removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1311 vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1312 if (interfaceManager.isExternalInterface(interfaceName)) {
1313 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1314 NwConstants.DEL_FLOW);
1316 if (!isBgpVpnInternetVpn) {
1317 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1319 LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1320 + " successful", interfaceName, dpId, vpnName);
1322 // Interface is retained in the DPN, but its Link Down.
1323 // Only withdraw the prefixes for this interface from BGP
1324 withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1328 private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1329 final String vpnName, final long vpnId, String gwMac,
1330 WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1331 final WriteTransaction writeInvTxn) {
1334 InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1335 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1336 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1337 Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1338 LogicalDatastoreType.OPERATIONAL, path);
1339 String primaryRd = vpnUtil.getVpnRd(vpnName);
1340 LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1341 interfaceName, dpnId, vpnName, primaryRd);
1342 if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1343 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1344 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1345 interfaceName, dpnId, vpnName, nextHops);
1346 for (Adjacency nextHop : nextHops) {
1347 if (nextHop.isPhysNetworkFunc()) {
1348 LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1349 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1350 fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1351 null/*writeCfgTxn*/);
1353 String rd = nextHop.getVrfId();
1354 List<String> nhList;
1355 if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1356 nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1358 // This is a primary adjacency
1359 nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1360 : Collections.emptyList();
1361 removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1362 interfaceName, writeInvTxn);
1364 if (!nhList.isEmpty()) {
1365 if (rd.equals(vpnName)) {
1366 //this is an internal vpn - the rd is assigned to the vpn instance name;
1367 //remove from FIB directly
1368 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1369 interfaceName, dpnId, writeConfigTxn));
1371 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1372 interfaceName, writeConfigTxn);
1375 LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1376 + " interface {}", nextHop.getIpAddress(), rd,
1377 nextHop.getAdjacencyType().toString(), interfaceName);
1378 bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1379 fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1382 String ip = nextHop.getIpAddress().split("/")[0];
1383 LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1384 if (vpnVipToPort != null) {
1385 vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1386 LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1387 + " for Interface {} ip {} on dpn {} for vpn {}",
1388 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1390 VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1391 if (vpnPortipToPort != null) {
1392 VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1393 LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1394 + "Interface {} ip {} on dpn {} for vpn {}",
1395 vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1399 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1400 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1401 + " Removing it.", interfaceName, vpnName, dpnId);
1402 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1404 } catch (ReadFailedException e) {
1405 LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1406 interfaceName, dpnId, vpnName);
1410 private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1411 String interfaceName, BigInteger dpnId,
1412 WriteTransaction writeConfigTxn) {
1414 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1416 LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1417 + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1418 vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1422 private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1423 BigInteger dpnId, String rd, String interfaceName,
1424 WriteTransaction writeConfigTxn) {
1425 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1426 vpnUtil.getVpnsImportingMyRoute(vpnName);
1427 nhList.forEach((nh) -> {
1428 //IRT: remove routes from other vpns importing it
1429 vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1430 nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1431 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1432 String vpnRd = vpn.getVrfId();
1433 if (vpnRd != null) {
1434 fibManager.removeOrUpdateFibEntry(vpnRd,
1435 nextHop.getIpAddress(), nh, writeConfigTxn);
1436 LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1437 + " prefix {} nextHop {} from VPN {} parentVpn {}"
1438 + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1439 vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1445 private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1446 int lportTag, String gwMac, String interfaceName,
1447 WriteTransaction writeInvTxn) {
1448 final Uuid subnetId = nextHop.getSubnetId();
1449 if (nextHop.getSubnetGatewayMacAddress() == null) {
1450 // A valid mac-address was not available for this subnet-gateway-ip
1451 // So a connected-mac-address was used for this subnet and we need
1452 // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1453 vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1455 arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1459 private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1460 String interfaceName) {
1461 // This is either an extra-route (or) a learned IP via subnet-route
1462 List<String> nhList = null;
1463 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1464 if (nextHopIp == null || nextHopIp.isEmpty()) {
1465 LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1466 + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1467 + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1469 nhList = Collections.emptyList();
1471 nhList = Collections.singletonList(nextHopIp);
1476 private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1477 VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1478 //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1479 // else use connected interface
1480 if (gwPort != null && gwPort.isSubnetIp()) {
1481 LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1482 gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1483 return Optional.of(gwPort.getMacAddress());
1485 return Optional.absent();
1489 protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1490 final VpnInterface update) {
1491 LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1492 update.getDpnId(), original.getVpnInstanceNames(),
1493 update.getVpnInstanceNames());
1494 final String vpnInterfaceName = update.getName();
1495 final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1496 final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1497 final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1498 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1499 final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1500 final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1501 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1503 LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1504 //handles switching between <internal VPN - external VPN>
1505 if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1506 LOG.info("update: handled VPNInterface {} on dpn {} update"
1507 + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1508 original.getName(), dpnId,
1509 VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1510 VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1513 for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1514 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1515 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1516 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1517 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1518 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1519 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1520 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1521 List<ListenableFuture<Void>> futures = new ArrayList<>();
1522 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1523 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1524 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1525 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1526 LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1527 update.getName(), newVpnName);
1528 //handle both addition and removal of adjacencies
1529 //currently, new adjacency may be an extra route
1530 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1531 if (!oldAdjs.equals(newAdjs)) {
1532 for (Adjacency adj : copyNewAdjs) {
1533 if (copyOldAdjs.contains(adj)) {
1534 copyOldAdjs.remove(adj);
1536 // add new adjacency - right now only extra route will hit this path
1537 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1538 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1539 dpnId, operTx, confTx);
1541 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1542 + " vpn interface {} on vpn {} dpnId {}",
1543 adj.getIpAddress(), adj.getNextHopIpList(),
1544 adj.getLabel(), adj.getSubnetId(), update.getName(),
1548 for (Adjacency adj : copyOldAdjs) {
1549 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1550 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1551 && !adj.isPhysNetworkFunc()) {
1552 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1554 Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1555 LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1556 if (optVpnInterface.isPresent()) {
1557 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1558 long vpnId = vpnUtil.getVpnId(newVpnName);
1559 vpnUtil.removePrefixToInterfaceAdj(adj, vpnId,
1560 vpnInterfaceOpDataEntry, operTx);
1562 LOG.info("update: Vpninterface {} not present in Operational",
1566 String vpnRd = vpnUtil.getVpnRd(newVpnName);
1567 LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1568 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1570 fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1571 if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1572 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1575 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1579 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1580 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1581 .getNextHopIpList(),
1582 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1587 for (ListenableFuture<Void> future : futures) {
1588 ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1589 update.getName(), update.getVpnInstanceNames());
1594 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1595 + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1600 private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1601 VpnInterface original, VpnInterface update) {
1602 boolean isSwap = Boolean.FALSE;
1603 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1604 final String interfaceName = key.getName();
1605 List<String> oldVpnList = original.getVpnInstanceNames().stream()
1606 .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1607 List<String> oldVpnListCopy = new ArrayList<>();
1608 oldVpnListCopy.addAll(oldVpnList);
1609 List<String> newVpnList = update.getVpnInstanceNames().stream()
1610 .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1611 oldVpnList.removeAll(newVpnList);
1612 newVpnList.removeAll(oldVpnListCopy);
1613 if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1614 for (String oldVpnName: oldVpnList) {
1615 isSwap = Boolean.TRUE;
1616 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1617 + " running config-driven swap removal", interfaceName, oldVpnName);
1618 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1619 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1620 + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1622 //Wait for previous interface bindings to be removed
1625 } catch (InterruptedException e) {
1628 for (String newVpnName: newVpnList) {
1629 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1630 isSwap = Boolean.TRUE;
1631 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1632 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1633 + "running config-driven swap addition", interfaceName, newVpnName);
1634 final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1635 final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1636 ? origAdjs.getAdjacency() : new ArrayList<>();
1637 final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1638 final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1639 ? updateAdjs.getAdjacency() : new ArrayList<>();
1641 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1642 LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1643 + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1644 interfaceName, oldVpnListCopy, newVpnName);
1651 private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1653 Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1654 synchronized (label.toString().intern()) {
1655 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1656 .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1657 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1658 LogicalDatastoreType.OPERATIONAL, lriIid);
1659 if (opResult.isPresent()) {
1660 LabelRouteInfo labelRouteInfo =
1661 new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1662 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1663 labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1666 LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1668 } catch (ReadFailedException e) {
1669 LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1671 } catch (TransactionCommitFailedException e) {
1672 LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1677 public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1678 SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1680 RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1681 VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1682 .addAugmentation(SubnetRoute.class, route).build();
1683 List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1684 InstanceIdentifierBuilder<VrfTables> idBuilder =
1685 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1686 InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1687 VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1688 if (writeConfigTxn != null) {
1689 writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1691 vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1693 LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1694 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1697 protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1698 Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1699 WriteTransaction writeConfigTxn) {
1700 String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1701 String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1703 Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1704 .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1705 if (optVpnInterface.isPresent()) {
1706 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1707 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1708 String vpnName = currVpnIntf.getVpnInstanceName();
1709 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1710 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1711 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1712 LogicalDatastoreType.OPERATIONAL, adjPath);
1713 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1714 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1715 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L : vpnInstanceOpData.getL3vni();
1716 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1717 List<Adjacency> adjacencies;
1718 if (optAdjacencies.isPresent()) {
1719 adjacencies = optAdjacencies.get().getAdjacency();
1721 // This code will be hit in case of first PNF adjacency
1722 adjacencies = new ArrayList<>();
1724 long vpnId = vpnUtil.getVpnId(vpnName);
1725 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1726 .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1727 Adjacency operationalAdjacency = null;
1728 //Handling dual stack neutron port primary adjacency
1729 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1730 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1731 currVpnIntf.getName(), vpnName);
1732 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1733 currVpnIntf.getName());
1734 if (interfaceState != null) {
1735 processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1736 currVpnIntf.getName(),
1737 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1740 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1741 && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1742 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1743 : RouteOrigin.STATIC;
1744 String nh = adj.getNextHopIpList().get(0);
1745 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1746 synchronized (vpnPrefixKey.intern()) {
1747 java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1748 vpnId, null, prefix, vpnName, nh, dpnId);
1749 if (rdToAllocate.isPresent()) {
1750 input.setRd(rdToAllocate.get());
1751 operationalAdjacency = populator.createOperationalAdjacency(input);
1752 int label = operationalAdjacency.getLabel().intValue();
1753 vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1754 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1755 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1756 LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1757 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1758 vpnName, label, currVpnIntf.getName(), dpnId);
1760 LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1764 // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1765 // Keeping the MPLS check for now.
1766 if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1767 final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1768 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1769 vpnUtil.getVpnsImportingMyRoute(vpnName);
1770 vpnsToImportRoute.forEach(vpn -> {
1771 if (vpn.getVrfId() != null) {
1772 vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1773 vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1775 rds -> vpnManager.addExtraRoute(
1776 vpnUtil.getVpnName(vpn.getVpnId()),
1777 adj.getIpAddress(), nh, rds,
1778 currVpnIntf.getVpnInstanceName(), l3vni,
1779 RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1780 opAdjacency, encapType, writeConfigTxn));
1785 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1786 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1787 currVpnIntf.getName(), vpnName);
1789 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1792 LogicalDatastoreType.OPERATIONAL,
1793 VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1794 prefix), VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(),
1795 prefix, adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1797 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1798 adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1799 0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1801 input.setRd(adj.getVrfId());
1803 if (operationalAdjacency == null) {
1804 operationalAdjacency = populator.createOperationalAdjacency(input);
1806 adjacencies.add(operationalAdjacency);
1807 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1808 VpnInterfaceOpDataEntry newVpnIntf =
1809 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1810 aug, dpnId, currVpnIntf.getLportTag(),
1811 currVpnIntf.getGatewayMacAddress());
1813 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1815 } catch (ReadFailedException e) {
1816 LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1817 + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1821 private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1822 Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1823 return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1826 protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1827 BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1828 String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1829 String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1831 Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1832 dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1833 if (optVpnInterface.isPresent()) {
1834 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1835 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1836 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1837 LogicalDatastoreType.OPERATIONAL, path);
1838 if (optAdjacencies.isPresent()) {
1839 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1841 if (!adjacencies.isEmpty()) {
1842 LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1843 Iterator<Adjacency> adjIt = adjacencies.iterator();
1844 while (adjIt.hasNext()) {
1845 Adjacency adjElem = adjIt.next();
1846 if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1847 String rd = adjElem.getVrfId();
1848 if (adj.getNextHopIpList() != null) {
1849 for (String nh : adj.getNextHopIpList()) {
1850 deleteExtraRouteFromCurrentAndImportingVpns(
1851 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1852 currVpnIntf.getName(), writeConfigTxn);
1854 } else if (adj.isPhysNetworkFunc()) {
1855 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1856 adj.getIpAddress(), adj.getSubnetId());
1857 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1865 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1866 dpnId, adj.getVrfId());
1868 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1869 + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1872 } catch (ReadFailedException e) {
1873 LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1874 adj.getIpAddress(), interfaceName, dpnId, vpnName);
1878 private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1879 String rd, String intfName, WriteTransaction writeConfigTxn) {
1880 vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1881 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1882 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1883 String vpnRd = vpn.getVrfId();
1884 if (vpnRd != null) {
1885 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1890 InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1891 return InstanceIdentifier.builder(NeutronRouterDpns.class)
1892 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1893 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1896 InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1897 return InstanceIdentifier.builder(NeutronRouterDpns.class)
1898 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1901 protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1902 WriteTransaction writeConfigTxn, String vpnName) {
1903 if (vpnInterface == null) {
1906 List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1908 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1909 + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1912 for (Adjacency adj : adjs) {
1913 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1914 String primaryInterfaceIp = adj.getIpAddress();
1915 String macAddress = adj.getMacAddress();
1916 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1918 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1919 VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1921 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1922 .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1923 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1924 routerInt, label, writeConfigTxn);
1925 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1926 + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1931 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1932 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1933 primaryRd, vpnName);
1936 protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1937 WriteTransaction writeConfigTxn, String vpnName) {
1938 Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1939 String rd = vpnUtil.getVpnRd(vpnName);
1941 List<Adjacency> adjsList = adjs.getAdjacency();
1942 for (Adjacency adj : adjsList) {
1943 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1944 String primaryInterfaceIp = adj.getIpAddress();
1945 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1946 fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1947 LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1948 + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1953 LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1954 vpnInterface.getName(), rd);
1958 private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1959 if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1960 LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1961 vpnName, intefaceData.vpnInterface.getName());
1964 final VpnInterfaceKey key = intefaceData.identifier
1965 .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1966 final String interfaceName = key.getName();
1967 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1968 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1969 addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1970 intefaceData.identifier, vpnName);
1974 private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1975 VpnInterface vpnInterface, String vpnName) {
1976 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1978 if (vpnInterfaces == null) {
1979 vpnInterfaces = new ConcurrentLinkedQueue<>();
1981 vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
1982 unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
1983 LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
1984 vpnInterface.getName(), vpnName);
1987 public boolean isVpnInstanceReady(String vpnInstanceName) {
1988 String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
1989 if (vpnRd == null) {
1992 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1994 return vpnInstanceOpDataEntry != null;
1997 public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
1998 synchronized (vpnInstanceName.intern()) {
1999 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2000 unprocessedVpnInterfaces.get(vpnInstanceName);
2001 if (vpnInterfaces != null) {
2002 while (!vpnInterfaces.isEmpty()) {
2003 UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2004 if (hasVpnInstanceCreatedSuccessfully) {
2005 processSavedInterface(savedInterface, vpnInstanceName);
2006 LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2007 savedInterface.vpnInterface.getName(), vpnInstanceName);
2009 LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2010 savedInterface.vpnInterface.getName(), vpnInstanceName);
2014 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2019 private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2020 VpnInterface vpnInterface) {
2021 synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2022 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2023 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2024 if (vpnInterfaces != null) {
2025 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2026 LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2027 + "unprocessed list", vpnInterface.getName(),
2028 VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2031 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2032 VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2037 public void vpnInstanceIsReady(String vpnInstanceName) {
2038 processSavedInterfaces(vpnInstanceName, true);
2041 public void vpnInstanceFailed(String vpnInstanceName) {
2042 processSavedInterfaces(vpnInstanceName, false);
2045 private static class UnprocessedVpnInterfaceData {
2046 InstanceIdentifier<VpnInterface> identifier;
2047 VpnInterface vpnInterface;
2049 UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2050 this.identifier = identifier;
2051 this.vpnInterface = vpnInterface;
2055 public int hashCode() {
2056 final int prime = 31;
2058 result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2059 result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2064 public boolean equals(Object obj) {
2071 if (getClass() != obj.getClass()) {
2074 UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2075 if (identifier == null) {
2076 if (other.identifier != null) {
2079 } else if (!identifier.equals(other.identifier)) {
2082 if (vpnInterface == null) {
2083 if (other.vpnInterface != null) {
2086 } else if (!vpnInterface.equals(other.vpnInterface)) {
2093 public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2094 String primaryRd = vpnUtil.getVpnRd(vpnName);
2095 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2096 if (vpnInstanceOpData == null) {
2099 List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2100 if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2103 LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2104 vpnToDpnLists.forEach(vpnToDpnList -> {
2105 if (vpnToDpnList.getVpnInterfaces() == null) {
2108 vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2110 InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2111 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2112 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2113 .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2114 if (!vpnInterfaceOptional.isPresent()) {
2117 List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2118 vpnInterface.getInterfaceName());
2119 if (configVpnAdjacencies == null) {
2120 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2123 List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2124 .augmentation(AdjacenciesOp.class).getAdjacency();
2125 // Due to insufficient rds, some of the extra route wont get processed when it is added.
2126 // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2127 // in operational DS. These unprocessed adjacencies will be handled below.
2128 // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2129 // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2130 configVpnAdjacencies.stream()
2131 .filter(adjacency -> operationVpnAdjacencies.stream()
2132 .noneMatch(operationalAdjacency ->
2133 operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2134 .forEach(adjacency -> {
2135 LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2136 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2138 // TODO Deal with sequencing — the config tx must only submitted
2139 // if the oper tx goes in
2140 if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2141 List<ListenableFuture<Void>> futures = new ArrayList<>();
2142 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2143 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2144 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2145 primaryRd, adjacency, vpnInterfaceOptional.get()
2146 .getDpnId(), confTx, operTx)))));
2149 return Collections.emptyList();
2153 } catch (ReadFailedException e) {
2154 LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2155 vpnName, primaryRd);
2161 private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2162 private final String interfaceName;
2163 private final boolean add;
2164 private final String txnDestination;
2166 PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2167 this.interfaceName = interfaceName;
2169 this.txnDestination = transactionDest;
2173 public void onSuccess(Void voidObj) {
2175 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2176 interfaceName, txnDestination);
2178 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2183 public void onFailure(Throwable throwable) {
2185 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2186 interfaceName, txnDestination, throwable);
2188 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2189 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);