2 * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.vpnmanager;
10 import static java.util.Collections.emptyList;
11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
15 import com.google.common.base.Optional;
16 import com.google.common.base.Preconditions;
17 import com.google.common.collect.Iterators;
18 import com.google.common.util.concurrent.FutureCallback;
19 import com.google.common.util.concurrent.Futures;
20 import com.google.common.util.concurrent.ListenableFuture;
21 import com.google.common.util.concurrent.MoreExecutors;
22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.HashSet;
26 import java.util.List;
28 import java.util.Objects;
30 import java.util.concurrent.ConcurrentHashMap;
31 import java.util.concurrent.ConcurrentLinkedQueue;
32 import java.util.concurrent.ExecutionException;
33 import java.util.concurrent.locks.ReentrantLock;
34 import java.util.function.Consumer;
35 import java.util.function.Predicate;
36 import java.util.stream.Collectors;
37 import javax.annotation.PostConstruct;
38 import javax.annotation.PreDestroy;
39 import javax.inject.Inject;
40 import javax.inject.Singleton;
41 import org.eclipse.jdt.annotation.Nullable;
42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
48 import org.opendaylight.genius.infra.Datastore.Configuration;
49 import org.opendaylight.genius.infra.Datastore.Operational;
50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
52 import org.opendaylight.genius.infra.TransactionAdapter;
53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
54 import org.opendaylight.genius.infra.TypedWriteTransaction;
55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
56 import org.opendaylight.genius.mdsalutil.NWUtil;
57 import org.opendaylight.genius.mdsalutil.NwConstants;
58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
60 import org.opendaylight.genius.utils.JvmGlobalLocks;
61 import org.opendaylight.infrautils.caches.CacheProvider;
62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
122 import org.opendaylight.yangtools.yang.common.Uint32;
123 import org.opendaylight.yangtools.yang.common.Uint64;
124 import org.slf4j.Logger;
125 import org.slf4j.LoggerFactory;
128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
130 private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
131 private static final short DJC_MAX_RETRIES = 3;
133 private final DataBroker dataBroker;
134 private final ManagedNewTransactionRunner txRunner;
135 private final IBgpManager bgpManager;
136 private final IFibManager fibManager;
137 private final IMdsalApiManager mdsalManager;
138 private final IdManagerService idManager;
139 private final OdlInterfaceRpcService ifaceMgrRpcService;
140 private final VpnFootprintService vpnFootprintService;
141 private final IInterfaceManager interfaceManager;
142 private final IVpnManager vpnManager;
143 private final ArpResponderHandler arpResponderHandler;
144 private final JobCoordinator jobCoordinator;
145 private final VpnUtil vpnUtil;
147 private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
149 private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
150 new ConcurrentHashMap<>();
152 private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
155 public VpnInterfaceManager(final DataBroker dataBroker,
156 final IBgpManager bgpManager,
157 final IdManagerService idManager,
158 final IMdsalApiManager mdsalManager,
159 final IFibManager fibManager,
160 final OdlInterfaceRpcService ifaceMgrRpcService,
161 final VpnFootprintService vpnFootprintService,
162 final IInterfaceManager interfaceManager,
163 final IVpnManager vpnManager,
164 final ArpResponderHandler arpResponderHandler,
165 final JobCoordinator jobCoordinator,
166 final CacheProvider cacheProvider,
167 final VpnUtil vpnUtil) {
168 super(VpnInterface.class, VpnInterfaceManager.class);
170 this.dataBroker = dataBroker;
171 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
172 this.bgpManager = bgpManager;
173 this.idManager = idManager;
174 this.mdsalManager = mdsalManager;
175 this.fibManager = fibManager;
176 this.ifaceMgrRpcService = ifaceMgrRpcService;
177 this.vpnFootprintService = vpnFootprintService;
178 this.interfaceManager = interfaceManager;
179 this.vpnManager = vpnManager;
180 this.arpResponderHandler = arpResponderHandler;
181 this.jobCoordinator = jobCoordinator;
182 this.vpnUtil = vpnUtil;
184 vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
185 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
186 VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
189 public Runnable isNotifyTaskQueued(String intfName) {
190 return vpnIntfMap.remove(intfName);
194 public void start() {
195 LOG.info("{} start", getClass().getSimpleName());
196 registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
201 public void close() {
203 vpnInstanceOpDataEntryCache.close();
207 protected InstanceIdentifier<VpnInterface> getWildCardPath() {
208 return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
212 protected VpnInterfaceManager getDataTreeChangeListener() {
213 return VpnInterfaceManager.this;
217 public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
218 LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
219 LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
220 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
221 addVpnInterface(identifier, vpnInterface, null, null);
224 private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
225 final VpnInterface vpnInterface, String vpnName) {
226 // FIXME: separate this out somehow?
227 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
230 if (isVpnInstanceReady(vpnName)) {
233 addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
240 // TODO Clean up the exception handling
241 @SuppressWarnings("checkstyle:IllegalCatch")
242 private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
243 final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
244 for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
245 String vpnName = vpnInterfaceVpnInstance.getVpnName();
246 addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
250 private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
251 final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
252 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
253 final String interfaceName = key.getName();
255 if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
256 LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
257 vpnName, vpnInterface.getName());
260 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
261 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
262 List<Adjacency> copyOldAdjs = null;
263 if (oldAdjs != null) {
264 copyOldAdjs = new ArrayList<>();
265 copyOldAdjs.addAll(oldAdjs);
267 List<Adjacency> copyNewAdjs = null;
268 if (newAdjs != null) {
269 copyNewAdjs = new ArrayList<>();
270 copyNewAdjs.addAll(newAdjs);
272 addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
275 private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
276 final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
277 final @Nullable List<Adjacency> newAdjs,
278 final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
279 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
280 final String interfaceName = key.getName();
281 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
282 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
283 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
284 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
285 if (interfaceState != null) {
287 final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
288 final int ifIndex = interfaceState.getIfIndex();
289 jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
290 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
291 // (the inventory tx goes in last)
292 List<ListenableFuture<Void>> futures = new ArrayList<>();
293 //set of prefix used, as entry in prefix-to-interface datastore
294 // is prerequisite for refresh Fib to avoid race condition leading to
295 // missing remote next hop in bucket actions on bgp-vpn delete
296 Set<String> prefixListForRefreshFib = new HashSet<>();
297 ListenableFuture<Void> confFuture =
298 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
299 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
300 operTx -> futures.add(
301 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
303 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
305 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
306 processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
307 confTx, operTx, invTx, interfaceState, vpnName,
308 prefixListForRefreshFib);
309 if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
310 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
311 + " Update for swapping VPN {} case.", interfaceName, vpnName);
312 if (newAdjs != null) {
313 for (Adjacency adj : newAdjs) {
314 if (oldAdjs.contains(adj)) {
317 if (!isBgpVpnInternetVpn
318 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
319 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
320 primaryRd, adj, dpnId, operTx, confTx, invTx,
321 prefixListForRefreshFib);
326 for (Adjacency adj : oldAdjs) {
327 if (!isBgpVpnInternetVpn
328 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
329 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
335 Futures.addCallback(confFuture,
336 new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
337 MoreExecutors.directExecutor());
338 futures.add(confFuture);
339 Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
340 MoreExecutors.directExecutor());
341 LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
342 + " processed successfully", interfaceName, vpnName, dpnId);
345 } catch (NumberFormatException | IllegalStateException e) {
346 LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
347 + "interface {}. Interface addition on vpn {} failed", interfaceName,
351 } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
352 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
354 ListenableFuture<Void> future =
355 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
356 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
358 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
359 vpnName, vpnInterface.getDpnId());
361 ListenableFutures.addErrorLogging(future, LOG,
362 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
363 return Collections.singletonList(future);
366 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
367 + " is not available", interfaceName, vpnName);
370 LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
371 + " as vpn is pending delete", interfaceName, vpnName,
372 vpnInterface.getDpnId());
376 // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
377 // see comments below.
378 @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
379 protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
380 final int lportTag, boolean isInterfaceUp,
381 TypedWriteTransaction<Configuration> writeConfigTxn,
382 TypedWriteTransaction<Operational> writeOperTxn,
383 TypedReadWriteTransaction<Configuration> writeInvTxn,
384 Interface interfaceState, final String vpnName,
385 Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
386 final String interfaceName = vpnInterface.getName();
387 Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
389 VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
390 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
391 if (!isInterfaceUp) {
392 LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
393 interfaceName, dpId, vpnName);
394 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
395 if (VpnConstants.INVALID_ID.equals(vpnId)) {
396 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
397 + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
402 boolean waitForVpnInterfaceOpRemoval = false;
403 if (opVpnInterface != null) {
404 String opVpnName = opVpnInterface.getVpnInstanceName();
405 String primaryInterfaceIp = null;
406 if (Objects.equals(opVpnName, vpnName)) {
407 // Please check if the primary VRF Entry does not exist for VPNInterface
408 // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
410 // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
411 List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
413 LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
414 + " for this vpn interface could not be obtained", interfaceName, dpId,
418 for (Adjacency adj : adjs) {
419 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
420 primaryInterfaceIp = adj.getIpAddress();
424 if (primaryInterfaceIp == null) {
425 LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
426 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
430 // Get the rd of the vpn instance
431 VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
433 LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
434 + " bailing out from here.", interfaceName, dpId, vpnName);
437 waitForVpnInterfaceOpRemoval = true;
439 LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
440 + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
443 if (!waitForVpnInterfaceOpRemoval) {
444 // Add the VPNInterface and quit
445 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
446 null/*ipAddressSourceValuePair*/,
448 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
449 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
450 if (!isBgpVpnInternetVpn) {
451 vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
453 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
455 if (interfaceManager.isExternalInterface(interfaceName)) {
456 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
457 NwConstants.ADD_FLOW);
462 // FIB didn't get a chance yet to clean up this VPNInterface
463 // Let us give it a chance here !
464 LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
465 + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
467 Runnable notifyTask = new VpnNotifyTask();
468 synchronized (notifyTask) {
469 // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
470 // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
471 // published here while holding the lock on it so this path will hit the wait before notify can be
473 vpnIntfMap.put(interfaceName, notifyTask);
475 notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
476 } catch (InterruptedException e) {
481 vpnIntfMap.remove(interfaceName);
484 if (opVpnInterface != null) {
485 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
486 + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
488 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
491 // VPNInterface got removed, proceed with Add
492 LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
493 interfaceName, dpId, vpnName);
494 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
495 null/*ipAddressSourceValuePair*/,
497 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
498 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
499 if (!isBgpVpnInternetVpn) {
500 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
502 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
503 + " FIB to clean up", interfaceName, dpId, vpnName);
504 if (interfaceManager.isExternalInterface(interfaceName)) {
505 processExternalVpnInterface(interfaceName, vpnName, dpId,
506 lportTag, NwConstants.ADD_FLOW);
510 // Interface is retained in the DPN, but its Link Up.
511 // Advertise prefixes again for this interface to BGP
512 InstanceIdentifier<VpnInterface> identifier =
513 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
514 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
515 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
516 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
517 // Perform similar operation as interface add event for extraroutes.
518 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
519 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
520 LogicalDatastoreType.CONFIGURATION, path);
521 if (!optAdjacencies.isPresent()) {
522 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
525 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
526 for (Adjacency adjacency : adjacencies) {
527 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
530 // if BGPVPN Internet, filter only IPv6 Adjacencies
531 if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
534 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
535 dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
537 } catch (ReadFailedException e) {
538 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
539 interfaceName, vpnName, primaryRd, dpId);
544 private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
545 int lportTag, int addOrRemove) {
548 // vpn instance of ext-net interface is the network-id
549 extNetworkId = new Uuid(vpnName);
550 } catch (IllegalArgumentException e) {
551 LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
552 + " on dpn {} failed", vpnName, interfaceName, dpId);
556 List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
557 if (routerIds == null || routerIds.isEmpty()) {
558 LOG.info("processExternalVpnInterface: No router is associated with {}."
559 + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
560 extNetworkId.getValue(), interfaceName, dpId, vpnName);
564 LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
565 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
566 for (Uuid routerId : routerIds) {
567 String routerName = routerId.getValue();
568 Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
569 if (Objects.equals(primarySwitch, dpId)) {
570 Routers router = vpnUtil.getExternalRouter(routerName);
571 if (router != null) {
572 if (addOrRemove == NwConstants.ADD_FLOW) {
573 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
574 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
575 dpId, interfaceName, lportTag);
577 vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
578 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
579 dpId, interfaceName, lportTag);
582 LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
583 + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
584 interfaceName, dpId, vpnName);
590 // TODO Clean up the exception handling
591 @SuppressWarnings("checkstyle:IllegalCatch")
592 private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
593 final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
594 String vpnName, String interfaceName) {
596 LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
597 interfaceName, dpnId, vpnName);
600 if (rd.equals(vpnName)) {
601 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
602 + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
606 LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
607 interfaceName, dpnId, vpnName, rd);
609 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
610 if (nextHopIp == null) {
611 LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
612 + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
619 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
620 Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
621 LogicalDatastoreType.OPERATIONAL, path);
622 if (adjacencies.isPresent()) {
623 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
624 if (nextHops != null && !nextHops.isEmpty()) {
625 LOG.debug("advertiseAdjacenciesForVpnToBgp: NextHops are {} for interface {} on dpn {} for vpn {}"
626 + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
627 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
628 Uint32 l3vni = vpnInstanceOpData.getL3vni();
629 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
630 ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
631 for (Adjacency nextHop : nextHops) {
632 if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
635 String gatewayMac = null;
636 Uint32 label = Uint32.ZERO;
637 if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
638 final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
639 vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
640 gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
643 label = nextHop.getLabel();
646 LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
647 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
648 bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
649 encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
651 LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
652 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
653 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
654 } catch (Exception e) {
655 LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
656 + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
657 interfaceName, dpnId, e);
662 } catch (ReadFailedException e) {
663 LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
664 + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
668 // TODO Clean up the exception handling
669 @SuppressWarnings("checkstyle:IllegalCatch")
670 private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
671 String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
672 TypedWriteTransaction<Operational> writeOperTx) {
674 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
675 String rd = vpnUtil.getVpnRd(interfaceName);
677 LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
678 interfaceName, vpnName);
681 if (rd.equals(vpnName)) {
683 "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
684 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
688 LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
690 Optional<AdjacenciesOp> adjacencies = Optional.absent();
692 adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
694 } catch (ReadFailedException e) {
695 LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
696 interfaceName, vpnName);
698 if (adjacencies.isPresent()) {
699 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
701 if (nextHops != null && !nextHops.isEmpty()) {
702 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
703 nextHops, interfaceName, vpnName, rd);
704 for (Adjacency nextHop : nextHops) {
706 if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
707 LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
708 + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
709 interfaceName, vpnName);
710 bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
711 LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
712 + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
713 interfaceName, vpnName);
714 } else if (nextHop.getNextHopIpList() != null) {
715 // Perform similar operation as interface delete event for extraroutes.
716 String allocatedRd = nextHop.getVrfId();
717 for (String nh : nextHop.getNextHopIpList()) {
718 deleteExtraRouteFromCurrentAndImportingVpns(
719 vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
723 } catch (Exception e) {
724 LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
725 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
732 @SuppressWarnings("checkstyle:IllegalCatch")
733 protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
734 String primaryRd, String interfaceName, final Uint32 vpnId,
735 TypedWriteTransaction<Configuration> writeConfigTxn,
736 TypedWriteTransaction<Operational> writeOperTxn,
737 TypedReadWriteTransaction<Configuration> writeInvTxn,
738 Interface interfaceState, Set<String> prefixListForRefreshFib)
739 throws ExecutionException, InterruptedException {
740 InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
742 Optional<VpnInterface> vpnInteface = Optional.absent();
744 vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
745 LogicalDatastoreType.CONFIGURATION, identifier);
746 } catch (ReadFailedException e) {
747 LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
748 + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
750 Uuid intfnetworkUuid = null;
751 NetworkType networkType = null;
752 Long segmentationId = Long.valueOf(-1);
753 Adjacencies adjacencies = null;
754 if (vpnInteface.isPresent()) {
755 intfnetworkUuid = vpnInteface.get().getNetworkId();
756 networkType = vpnInteface.get().getNetworkType();
757 segmentationId = vpnInteface.get().getSegmentationId().toJava();
758 adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
759 if (adjacencies == null) {
760 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
761 null/*gwMac*/, null/*gatewayIp*/, writeOperTxn);
765 // Get the rd of the vpn instance
766 String nextHopIp = null;
767 String gatewayIp = null;
769 nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
770 } catch (Exception e) {
771 LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
772 + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
774 List<String> nhList = new ArrayList<>();
775 if (nextHopIp != null) {
776 nhList.add(nextHopIp);
777 LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
778 interfaceName, dpnId, vpnName, nhList);
780 Optional<String> gwMac = Optional.absent();
781 String vpnInterfaceSubnetGwMacAddress = null;
782 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
783 Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
784 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
785 VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
786 VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
787 List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
788 List<Adjacency> value = new ArrayList<>();
789 for (Adjacency nextHop : nextHops) {
790 String rd = primaryRd;
791 String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
792 if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
793 && NWUtil.isIpv4Address(nexthopIpValue)) {
794 String prefix = nextHop.getIpAddress() == null ? "null" :
795 VpnUtil.getIpPrefix(nextHop.getIpAddress());
796 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
797 + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
800 if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
801 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
802 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
803 ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
804 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
805 + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
807 Prefixes prefixes = intfnetworkUuid != null
808 ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
809 segmentationId, prefixCue) :
810 VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
811 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
812 vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
813 final Uuid subnetId = nextHop.getSubnetId();
815 gatewayIp = nextHop.getSubnetGatewayIp();
816 if (gatewayIp == null) {
817 Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
818 if (gatewayIpOptional.isPresent()) {
819 gatewayIp = gatewayIpOptional.get();
823 if (gatewayIp != null) {
824 gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
825 if (gwMac.isPresent()) {
826 // A valid mac-address is available for this subnet-gateway-ip
827 // Use this for programming ARP_RESPONDER table here. And save this
828 // info into vpnInterface operational, so it can used in VrfEntryProcessor
829 // to populate L3_GW_MAC_TABLE there.
830 arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
831 gatewayIp, gwMac.get());
832 vpnInterfaceSubnetGwMacAddress = gwMac.get();
834 // A valid mac-address is not available for this subnet-gateway-ip
835 // Use the connected-mac-address to configure ARP_RESPONDER Table.
836 // Save this connected-mac-address as gateway-mac-address for the
837 // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
838 gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
839 if (gwMac.isPresent()) {
840 vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
841 NwConstants.ADD_FLOW, gwMac.get());
842 arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
843 gatewayIp, gwMac.get());
845 LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
846 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
848 subnetId, interfaceName, vpnName, gatewayIp);
852 LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
853 + "cannot create ARP responder flow for interface name {}, vpnName {}",
854 subnetId, interfaceName, vpnName);
855 gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
857 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
858 + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
860 //Extra route adjacency
861 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
862 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
863 // FIXME: separate this out somehow?
864 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
867 java.util.Optional<String> rdToAllocate = vpnUtil
868 .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
869 nextHop.getNextHopIpList().get(0), dpnId);
870 if (rdToAllocate.isPresent()) {
871 rd = rdToAllocate.get();
872 LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
875 LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
881 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
882 + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
883 interfaceName, dpnId);
885 // Please note that primary adjacency will use a subnet-gateway-mac-address that
886 // can be different from the gateway-mac-address within the VRFEntry as the
887 // gateway-mac-address is a superset.
888 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
889 L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
890 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
891 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
892 Adjacency operationalAdjacency = null;
894 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
895 } catch (NullPointerException e) {
896 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
897 input, e.getMessage());
900 if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
901 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
902 vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
905 value.add(operationalAdjacency);
908 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
909 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
910 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
912 L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
913 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
914 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
916 for (Adjacency nextHop : aug.getAdjacency()) {
917 // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
918 if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
919 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
920 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
921 registeredPopulator.populateFib(input, writeConfigTxn);
926 private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
927 long lportTag, String gwMac, String gwIp,
928 TypedWriteTransaction<Operational> writeOperTxn) {
929 VpnInterfaceOpDataEntry opInterface =
930 VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
931 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
932 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
933 writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
934 LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
935 interfaceName, dpnId, vpnName);
938 // TODO Clean up the exception handling
939 @SuppressWarnings("checkstyle:IllegalCatch")
940 public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
941 StateTunnelList stateTunnelList,
942 TypedWriteTransaction<Configuration> writeConfigTxn,
943 TypedWriteTransaction<Operational> writeOperTxn) {
945 String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
946 Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
947 AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
948 List<Adjacency> adjList =
949 adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
950 if (adjList.isEmpty()) {
951 LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
952 vpnInterface, srcDpnId);
955 String prefix = null;
956 List<Adjacency> value = new ArrayList<>();
957 boolean isNextHopAddReqd = false;
958 String vpnName = vpnInterface.getVpnInstanceName();
959 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
960 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
961 LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
962 vpnInterface.getName(), vpnInterface.getDpnId(),
963 vpnInterface.getVpnInstanceName(), adjList);
964 for (Adjacency adj : adjList) {
965 String rd = adj.getVrfId();
966 rd = rd != null ? rd : vpnName;
967 prefix = adj.getIpAddress();
968 Uint32 label = adj.getLabel();
969 List<String> nhList = Collections.singletonList(srcTepIp);
970 List<String> nextHopList = adj.getNextHopIpList();
971 // If TEP is added , update the nexthop of primary adjacency.
972 // Secondary adj nexthop is already pointing to primary adj IP address.
973 if (nextHopList == null || nextHopList.isEmpty()) {
974 isNextHopAddReqd = true;
977 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
978 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
980 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
981 if (!vrfEntryOptional.isPresent()) {
984 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
985 if (!nhList.contains(srcTepIp)) {
986 nhList.add(srcTepIp);
987 isNextHopAddReqd = true;
992 if (isNextHopAddReqd) {
993 updateLabelMapper(label, nhList);
994 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
995 + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
996 vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
997 // Update the VRF entry with nextHop
998 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
999 label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1001 //Get the list of VPN's importing this route(prefix) .
1002 // Then update the VRF entry with nhList
1003 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1004 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1005 String vpnRd = vpn.getVrfId();
1006 if (vpnRd != null) {
1007 fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1008 srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1009 LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1010 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1011 vpnInterface.getName(), srcDpnId, vpnName,
1012 vpn.getVpnInstanceName(), vpnRd);
1015 // Advertise the prefix to BGP only for external vpn
1016 // since there is a nexthop change.
1018 if (!rd.equalsIgnoreCase(vpnName)) {
1019 bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1020 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1021 null /*gatewayMacAddress*/);
1023 LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1024 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1026 } catch (Exception ex) {
1027 LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1028 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1029 vpnInterface.getName(), srcDpnId, vpnName, ex);
1033 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1034 VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1035 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1036 .addAugmentation(AdjacenciesOp.class, aug).build();
1037 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1038 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1039 writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1040 LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1041 vpnInterface.getName(), srcDpnId, vpnName);
1045 // TODO Clean up the exception handling
1046 @SuppressWarnings("checkstyle:IllegalCatch")
1047 public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1048 StateTunnelList stateTunnelList,
1049 TypedWriteTransaction<Configuration> writeConfigTxn,
1050 TypedWriteTransaction<Operational> writeOperTxn) {
1052 AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1053 List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1054 String prefix = null;
1055 boolean isNextHopRemoveReqd = false;
1056 String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1057 Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1058 String vpnName = vpnInterface.getVpnInstanceName();
1059 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1060 String primaryRd = vpnUtil.getVpnRd(vpnName);
1061 if (adjList != null) {
1062 List<Adjacency> value = new ArrayList<>();
1063 LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1064 vpnInterface.getName(), vpnInterface.getDpnId(),
1065 vpnInterface.getVpnInstanceName(), adjList);
1066 for (Adjacency adj : adjList) {
1067 List<String> nhList = new ArrayList<>();
1068 String rd = adj.getVrfId();
1069 rd = rd != null ? rd : vpnName;
1070 prefix = adj.getIpAddress();
1071 List<String> nextHopList = adj.getNextHopIpList();
1072 Uint32 label = adj.getLabel();
1073 if (nextHopList != null && !nextHopList.isEmpty()) {
1074 isNextHopRemoveReqd = true;
1076 // If TEP is deleted , remove the nexthop from primary adjacency.
1077 // Secondary adj nexthop will continue to point to primary adj IP address.
1078 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1079 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1081 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1082 if (!vrfEntryOptional.isPresent()) {
1085 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1086 if (nhList.contains(srcTepIp)) {
1087 nhList.remove(srcTepIp);
1088 isNextHopRemoveReqd = true;
1093 if (isNextHopRemoveReqd) {
1094 updateLabelMapper(label, nhList);
1095 LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1096 + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1097 prefix, nhList, vpnName,
1098 vpnId, rd, vpnInterface.getName());
1099 // Update the VRF entry with removed nextHop
1100 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1101 label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1103 //Get the list of VPN's importing this route(prefix) .
1104 // Then update the VRF entry with nhList
1105 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1106 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1107 String vpnRd = vpn.getVrfId();
1108 if (vpnRd != null) {
1109 fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1110 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1111 LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1112 + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1113 nhList, label, vpnInterface.getName(), srcDpnId,
1115 vpn.getVpnInstanceName(), vpnRd);
1119 // Withdraw prefix from BGP only for external vpn.
1121 if (!rd.equalsIgnoreCase(vpnName)) {
1122 bgpManager.withdrawPrefix(rd, prefix);
1124 LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1125 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1126 vpnInterface.getName(), srcDpnId,
1128 } catch (Exception ex) {
1129 LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1130 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1131 vpnInterface.getName(), srcDpnId, vpnName, ex);
1135 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1136 VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1137 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1138 .addAugmentation(AdjacenciesOp.class, aug).build();
1139 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1140 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1141 writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1142 LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1143 vpnInterface.getName(), srcDpnId, vpnName);
1147 private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1148 List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1150 String vpnRd = vpnUtil.getVpnRd(vpnName);
1151 final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1152 if (vpnInstanceOpDataEntry == null) {
1153 LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1154 + " to check for vpns exporting the routes", vpnName);
1155 return vpnsToExportRoute;
1158 Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1159 if (input.getVpnInstanceName() == null) {
1160 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {} without a name",
1164 return !input.getVpnInstanceName().equals(vpnName);
1167 Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1168 Iterable<String> commonRTs =
1169 VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1170 VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1171 return Iterators.size(commonRTs.iterator()) > 0;
1175 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1176 Collectors.toList());
1177 return vpnsToExportRoute;
1180 // TODO Clean up the exception handling
1181 @SuppressWarnings("checkstyle:IllegalCatch")
1182 void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1183 List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1184 for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1185 List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1186 if (vrfEntries != null) {
1187 ListenableFutures.addErrorLogging(
1188 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1189 for (VrfEntry vrfEntry : vrfEntries) {
1191 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1192 RouteOrigin.value(vrfEntry.getOrigin()))) {
1193 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1194 + " is not a controller managed non intervpn link route. Ignoring.",
1195 vpn.getVrfId(), vrfEntry.getDestPrefix());
1198 String prefix = vrfEntry.getDestPrefix();
1199 String gwMac = vrfEntry.getGatewayMacAddress();
1200 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1201 String nh = routePath.getNexthopAddress();
1202 Uint32 label = routePath.getLabel();
1203 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1204 vrfEntry.getOrigin()))) {
1206 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1207 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1208 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1209 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1210 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1211 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1214 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1215 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1216 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1217 SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1218 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1222 } catch (RuntimeException e) {
1223 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1224 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1225 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1228 }), LOG, "Error handing VPN exporting routes");
1230 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1231 vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1237 public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1238 LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1239 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1240 final String interfaceName = key.getName();
1241 for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1242 String vpnName = vpnInterfaceVpnInstance.getVpnName();
1243 removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1247 private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1248 final VpnInterface vpnInterface, final String vpnName,
1249 final String interfaceName) {
1250 if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1251 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1252 ListenableFuture<Void> future =
1253 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1254 deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1255 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1257 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1258 vpnInterface.getName(), vpnName);
1259 return Collections.singletonList(future);
1260 }, DJC_MAX_RETRIES);
1262 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1263 removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1267 @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1268 private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1269 final VpnInterface vpnInterface, final String vpnName,
1270 final String interfaceName, final Interface interfaceState) {
1271 LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1272 vpnName, vpnInterface.getDpnId());
1273 removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1274 jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1276 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1277 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1278 writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1279 writeOperTxn -> futures.add(
1280 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1281 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1282 interfaceName, vpnName);
1285 String gwMacAddress;
1286 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1287 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1288 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1290 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1291 LogicalDatastoreType.OPERATIONAL, interfaceId);
1292 } catch (ReadFailedException e) {
1293 LOG.error("remove: Failed to read data store for interface {} vpn {}",
1294 interfaceName, vpnName);
1297 if (interfaceState != null) {
1299 dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1300 } catch (NumberFormatException | IllegalStateException e) {
1301 LOG.error("remove: Unable to retrieve dpnId from interface operational"
1302 + " data store for interface {} on dpn {} for vpn {} Fetching"
1303 + " from vpn interface op data store. ", interfaceName,
1304 vpnInterface.getDpnId(), vpnName, e);
1307 ifIndex = interfaceState.getIfIndex();
1308 gwMacAddress = interfaceState.getPhysAddress().getValue();
1310 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1311 + " from vpn interface op.", interfaceName);
1312 if (optVpnInterface.isPresent()) {
1313 VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1314 dpId = vpnOpInterface.getDpnId();
1315 ifIndex = vpnOpInterface.getLportTag().intValue();
1316 gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1318 LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1319 + " as interfaceState and vpn interface op is not"
1320 + " available", interfaceName, vpnName);
1324 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1325 optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1326 writeConfigTxn, writeOperTxn, writeInvTxn);
1328 "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1330 interfaceName, vpnInterface.getDpnId(), vpnName);
1332 futures.add(configFuture);
1333 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1334 interfaceName, false, "Config"), MoreExecutors.directExecutor());
1336 }, DJC_MAX_RETRIES);
1339 protected void processVpnInterfaceDown(Uint64 dpId,
1340 String interfaceName,
1343 VpnInterfaceOpDataEntry vpnOpInterface,
1344 boolean isInterfaceStateDown,
1345 TypedWriteTransaction<Configuration> writeConfigTxn,
1346 TypedWriteTransaction<Operational> writeOperTxn,
1347 TypedReadWriteTransaction<Configuration> writeInvTxn)
1348 throws ExecutionException, InterruptedException {
1349 if (vpnOpInterface == null) {
1350 LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1351 + " as it is not available in operational data store", interfaceName, dpId);
1354 final String vpnName = vpnOpInterface.getVpnInstanceName();
1355 InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1356 interfaceName, vpnName);
1357 if (!isInterfaceStateDown) {
1358 final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1359 vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName, null);
1360 final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1361 removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1362 vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1363 if (interfaceManager.isExternalInterface(interfaceName)) {
1364 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1365 NwConstants.DEL_FLOW);
1367 if (!isBgpVpnInternetVpn) {
1368 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1370 LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1371 + " successful", interfaceName, dpId, vpnName);
1373 // Interface is retained in the DPN, but its Link Down.
1374 // Only withdraw the prefixes for this interface from BGP
1375 withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1379 private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1380 final String vpnName, final Uint32 vpnId, String gwMac,
1381 TypedWriteTransaction<Configuration> writeConfigTxn,
1382 TypedWriteTransaction<Operational> writeOperTxn,
1383 TypedReadWriteTransaction<Configuration> writeInvTxn)
1384 throws ExecutionException, InterruptedException {
1387 InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1388 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1389 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1390 SingleTransactionDataBroker.syncReadOptional(dataBroker,
1391 LogicalDatastoreType.OPERATIONAL, identifier);
1392 boolean isLearntIP = Boolean.FALSE;
1393 String primaryRd = vpnUtil.getVpnRd(vpnName);
1394 LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1395 interfaceName, dpnId, vpnName, primaryRd);
1396 if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1397 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1398 + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1401 AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1403 if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) {
1404 List<Adjacency> nextHops = adjacencies.getAdjacency();
1405 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1406 interfaceName, dpnId, vpnName, nextHops);
1407 for (Adjacency nextHop : nextHops) {
1408 if (nextHop.isPhysNetworkFunc()) {
1409 LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1410 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1411 fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1412 null, null/*writeCfgTxn*/);
1414 String rd = nextHop.getVrfId();
1415 List<String> nhList;
1416 if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1417 nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1418 isLearntIP = nextHop.getAdjacencyType() == AdjacencyType.LearntIp ? true : false;
1420 // This is a primary adjacency
1421 nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1423 removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1424 vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1425 interfaceName, writeInvTxn);
1427 if (!nhList.isEmpty()) {
1428 if (Objects.equals(primaryRd, vpnName)) {
1429 //this is an internal vpn - the rd is assigned to the vpn instance name;
1430 //remove from FIB directly
1431 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1432 interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1434 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1435 interfaceName, writeConfigTxn, writeOperTxn);
1438 LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1439 + " interface {}", nextHop.getIpAddress(), rd,
1440 nextHop.getAdjacencyType().toString(), interfaceName);
1441 bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1442 fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1445 String ip = nextHop.getIpAddress().split("/")[0];
1446 LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1447 if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1448 vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1449 LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1450 + " for Interface {} ip {} on dpn {} for vpn {}",
1451 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1453 // Remove the MIP-IP from VpnPortIpToPort.
1455 VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1456 if (persistedIp != null && persistedIp.isLearntIp()
1457 && persistedIp.getPortName().equals(interfaceName)) {
1458 VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1460 "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1461 + "from VpnPortipToPort",
1462 persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1465 VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1466 if (vpnPortipToPort != null) {
1467 VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1468 LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1469 + "Interface {} ip {} on dpn {} for vpn {}",
1470 vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1474 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1475 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1476 + " Removing it.", interfaceName, vpnName, dpnId);
1477 writeOperTxn.delete(identifier);
1479 } catch (ReadFailedException e) {
1480 LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1481 interfaceName, dpnId, vpnName);
1485 private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1486 String interfaceName, Uint64 dpnId,
1487 TypedWriteTransaction<Configuration> writeConfigTxn,
1488 TypedWriteTransaction<Operational> writeOperTx) {
1490 String primaryRd = vpnUtil.getVpnRd(vpnName);
1491 String prefix = nextHop.getIpAddress();
1492 String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1493 LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1494 nextHop, vpnName, interfaceName, dpnId);
1495 // FIXME: separate this out somehow?
1496 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1499 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1500 prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1501 //If extra-route is present behind at least one VM, then do not remove or update
1502 //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1503 //prefixtointerface DS
1506 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1511 LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1512 + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1513 vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1517 private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1518 Uint64 dpnId, String rd, String interfaceName,
1519 TypedWriteTransaction<Configuration> writeConfigTxn,
1520 TypedWriteTransaction<Operational> writeOperTx) {
1521 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1522 vpnUtil.getVpnsImportingMyRoute(vpnName);
1523 nhList.forEach((nh) -> {
1524 //IRT: remove routes from other vpns importing it
1525 vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1526 nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1527 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1528 String vpnRd = vpn.getVrfId();
1529 if (vpnRd != null) {
1530 fibManager.removeOrUpdateFibEntry(vpnRd,
1531 nextHop.getIpAddress(), nh, writeConfigTxn);
1532 LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1533 + " prefix {} nextHop {} from VPN {} parentVpn {}"
1534 + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1535 vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1541 private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1542 int lportTag, String gwMac, String gwIp, String interfaceName,
1543 TypedReadWriteTransaction<Configuration> writeInvTxn)
1544 throws ExecutionException, InterruptedException {
1545 final Uuid subnetId = nextHop.getSubnetId();
1546 if (nextHop.getSubnetGatewayMacAddress() == null) {
1547 // A valid mac-address was not available for this subnet-gateway-ip
1548 // So a connected-mac-address was used for this subnet and we need
1549 // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1550 vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1552 arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1556 private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1557 String interfaceName) {
1558 // This is either an extra-route (or) a learned IP via subnet-route
1559 List<String> nhList = null;
1560 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1561 if (nextHopIp == null || nextHopIp.isEmpty()) {
1562 LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1563 + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1564 + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1566 nhList = emptyList();
1568 nhList = Collections.singletonList(nextHopIp);
1573 private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1574 VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1575 //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1576 // else use connected interface
1577 if (gwPort != null && gwPort.isSubnetIp()) {
1578 LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1579 gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1580 return Optional.of(gwPort.getMacAddress());
1582 return Optional.absent();
1586 protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1587 final VpnInterface update) {
1588 LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1589 LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1590 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1591 if (original.equals(update)) {
1592 LOG.info("update: original {} update {} are same. No update required.", original, update);
1595 final String vpnInterfaceName = update.getName();
1596 final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1597 LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1598 //handles switching between <internal VPN - external VPN>
1599 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1600 List<ListenableFuture<Void>> futures = new ArrayList<>();
1601 if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1602 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1603 + "to newVpn(s) {}",
1604 original.getName(), dpnId,
1605 VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1606 VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1609 updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1614 private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1615 VpnInterface original, VpnInterface update,
1616 List<ListenableFuture<Void>> futures) {
1617 boolean isVpnInstanceUpdate = false;
1618 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1619 final String interfaceName = key.getName();
1620 List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1621 List<String> oldVpnListCopy = new ArrayList<>();
1622 oldVpnListCopy.addAll(oldVpnList);
1623 List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1624 List<String> newVpnListCopy = new ArrayList<>();
1625 newVpnListCopy.addAll(newVpnList);
1627 oldVpnList.removeAll(newVpnList);
1628 newVpnList.removeAll(oldVpnListCopy);
1629 //This block will execute only on if there is a change in the VPN Instance.
1630 if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1632 * Internet BGP-VPN Instance update with single router:
1633 * ====================================================
1634 * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1635 * 1st VPN Instance : router VPN or external BGP-VPN.
1636 * 2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1638 * VPN Instance UPDATE:
1639 * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1640 * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1642 * External BGP-VPN Instance update with single router:
1643 * ====================================================
1644 * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1646 * Updated VPN Instance will be always either internal router VPN to
1647 * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1649 * VPN Instance UPDATE:
1650 * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1651 * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1653 * Dual Router VPN Instance Update:
1654 * ================================
1655 * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1657 * 1st VPN Instance : router VPN or external BGP-VPN-1.
1658 * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1659 * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1661 * Dual Router --> Associated with common external BGP-VPN Instance.
1662 * 1st router and 2nd router are getting associated with single External BGP-VPN
1663 * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1664 * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1665 * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1669 isVpnInstanceUpdate = true;
1670 if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1671 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1672 && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1673 //Identify the external BGP-VPN Instance and pass that value as newVpnList
1674 List<String> externalBgpVpnList = new ArrayList<>();
1675 for (String newVpnName : newVpnListCopy) {
1676 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1677 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1678 if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1679 .BgpvpnType.BGPVPNExternal) {
1680 externalBgpVpnList.add(newVpnName);
1684 //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1685 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1686 externalBgpVpnList, oldVpnListCopy, futures);
1688 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1689 && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1690 //Identify the router VPN Instance and pass that value as oldVpnList
1691 List<String> routerVpnList = new ArrayList<>();
1692 for (String newVpnName : newVpnListCopy) {
1693 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1694 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1695 if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1697 routerVpnList.add(newVpnName);
1701 //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1702 updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1703 newVpnList, oldVpnListCopy, futures);
1706 //Handle remaining use cases.
1707 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1708 oldVpnListCopy, futures);
1711 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1712 oldVpnListCopy, futures);
1715 return isVpnInstanceUpdate;
1718 private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1719 VpnInterface original, VpnInterface update, List<String> oldVpnList,
1720 List<String> newVpnList, List<String> oldVpnListCopy,
1721 List<ListenableFuture<Void>> futures) {
1722 final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1723 final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1724 ? origAdjs.getAdjacency() : new ArrayList<>();
1725 final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1726 final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1727 ? updateAdjs.getAdjacency() : new ArrayList<>();
1729 boolean isOldVpnRemoveCallExecuted = false;
1730 for (String oldVpnName : oldVpnList) {
1731 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1732 + "remove from vpnName {} ", interfaceName, oldVpnName);
1733 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1734 LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1735 + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1737 isOldVpnRemoveCallExecuted = true;
1739 //Wait for previous interface bindings to be removed
1740 if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1743 } catch (InterruptedException e) {
1744 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1747 for (String newVpnName : newVpnList) {
1748 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1749 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1750 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1751 + "onto vpnName {} ", interfaceName, newVpnName);
1752 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1753 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1754 + "from oldVpn(s) {} to newVpn {} ",
1755 interfaceName, oldVpnListCopy, newVpnName);
1756 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1758 * In Dual stack network, first V4 subnet only attached to router and router is associated
1759 * with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1760 * Later V6 subnet is added to router, at this point existing VPN interface will get updated
1761 * with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1762 * is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1763 * BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1764 * router VPN as well as Internet BGP-VPN.
1766 * addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1767 * updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1768 * update in existing router VPN instance.
1770 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1771 LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1772 + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1773 updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1776 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1777 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1778 interfaceName, oldVpnListCopy, newVpnName);
1783 // TODO Clean up the exception handling
1784 @SuppressWarnings("checkstyle:IllegalCatch")
1785 private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1786 String vpnInterfaceName,
1787 List<ListenableFuture<Void>> futures) {
1788 final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1789 final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1790 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1791 final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1792 final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1793 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1795 final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1796 for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1797 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1798 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1799 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1800 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1801 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1802 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1803 //set of prefix used as entry in prefix-to-interface datastore
1804 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1805 // in bucket actions on bgp-vpn delete
1806 Set<String> prefixListForRefreshFib = new HashSet<>();
1807 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1808 confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1810 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1811 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1812 LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1813 update.getName(), newVpnName);
1814 //handle both addition and removal of adjacencies
1815 // currently, new adjacency may be an extra route
1816 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1817 if (!oldAdjs.equals(newAdjs)) {
1818 for (Adjacency adj : copyNewAdjs) {
1819 if (copyOldAdjs.contains(adj)) {
1820 copyOldAdjs.remove(adj);
1822 // add new adjacency
1823 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1825 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1826 dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1827 } catch (RuntimeException e) {
1828 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1829 + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1832 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1833 + " added to vpn interface {} on vpn {} dpnId {}",
1834 adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1835 adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1838 for (Adjacency adj : copyOldAdjs) {
1839 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1840 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1841 && !adj.isPhysNetworkFunc()) {
1842 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1845 String vpnRd = vpnUtil.getVpnRd(newVpnName);
1846 LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1847 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1849 fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1850 if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1851 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1854 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1858 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1859 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1860 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1864 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1865 MoreExecutors.directExecutor());
1866 futures.add(configTxFuture);
1867 for (ListenableFuture<Void> future : futures) {
1868 ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1869 update.getName(), update.getVpnInstanceNames());
1872 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1873 + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1879 private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1880 final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1882 // FIXME: separate this out somehow?
1883 final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1886 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1887 .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1888 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1889 LogicalDatastoreType.OPERATIONAL, lriIid);
1890 if (opResult.isPresent()) {
1891 LabelRouteInfo labelRouteInfo =
1892 new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1893 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1894 labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1896 LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1898 } catch (ReadFailedException e) {
1899 LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1901 } catch (TransactionCommitFailedException e) {
1902 LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1909 public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1910 SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1912 RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1913 VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1914 .addAugmentation(SubnetRoute.class, route).build();
1915 List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1916 InstanceIdentifierBuilder<VrfTables> idBuilder =
1917 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1918 InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1919 VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1920 if (writeConfigTxn != null) {
1921 writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1923 vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1925 LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1926 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1929 protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1930 Adjacency adj, Uint64 dpnId,
1931 TypedWriteTransaction<Operational> writeOperTxn,
1932 TypedWriteTransaction<Configuration> writeConfigTxn,
1933 TypedReadWriteTransaction<Configuration> writeInvTxn,
1934 Set<String> prefixListForRefreshFib)
1935 throws ExecutionException, InterruptedException {
1936 String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1937 String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1939 Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1940 .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1941 if (optVpnInterface.isPresent()) {
1942 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1943 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1944 String vpnName = currVpnIntf.getVpnInstanceName();
1945 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1946 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1947 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1948 LogicalDatastoreType.OPERATIONAL, adjPath);
1949 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1950 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1951 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1952 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1953 List<Adjacency> adjacencies = new ArrayList<>();
1954 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1955 adjacencies.addAll(optAdjacencies.get().getAdjacency());
1957 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1958 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1959 .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1960 Adjacency operationalAdjacency = null;
1961 //Handling dual stack neutron port primary adjacency
1962 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1963 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1964 currVpnIntf.getName(), vpnName);
1965 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1966 currVpnIntf.getName());
1967 if (interfaceState != null) {
1968 processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1969 currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1970 prefixListForRefreshFib);
1973 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1974 && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1975 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1976 : RouteOrigin.STATIC;
1977 String nh = adj.getNextHopIpList().get(0);
1978 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1979 // FIXME: separate out to somehow?
1980 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1983 java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1984 vpnId, null, prefix, vpnName, nh, dpnId);
1985 if (rdToAllocate.isPresent()) {
1986 input.setRd(rdToAllocate.get());
1987 operationalAdjacency = populator.createOperationalAdjacency(input);
1988 int label = operationalAdjacency.getLabel().intValue();
1989 vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1990 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1991 currVpnIntf.getName(), operationalAdjacency, encapType,
1992 prefixListForRefreshFib, writeConfigTxn);
1993 LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1994 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1995 vpnName, label, currVpnIntf.getName(), dpnId);
1997 LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2001 // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2002 // Keeping the MPLS check for now.
2003 if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2004 final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2005 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2006 vpnUtil.getVpnsImportingMyRoute(vpnName);
2007 vpnsToImportRoute.forEach(vpn -> {
2008 if (vpn.getVrfId() != null) {
2009 vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2011 vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2013 rds -> vpnManager.addExtraRoute(
2014 vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2015 nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2016 RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2017 encapType, prefixListForRefreshFib, writeConfigTxn));
2024 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2025 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2026 currVpnIntf.getName(), vpnName);
2028 InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2029 .getVpnInterfaceIdentifier(currVpnIntf.getName());
2030 Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2031 LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2032 Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2033 Prefixes.PrefixCue.PhysNetFunc);
2034 if (vpnIntefaceConfig.isPresent()) {
2035 pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2036 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2037 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2040 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2043 VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2044 prefix), pnfPrefix, true);
2046 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2047 adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2048 Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2049 RouteOrigin.LOCAL, writeConfigTxn);
2051 input.setRd(adj.getVrfId());
2053 if (operationalAdjacency == null) {
2054 operationalAdjacency = populator.createOperationalAdjacency(input);
2056 adjacencies.add(operationalAdjacency);
2057 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2058 VpnInterfaceOpDataEntry newVpnIntf =
2059 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2060 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2061 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2062 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2064 } catch (ReadFailedException e) {
2065 LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2066 + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2071 private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2072 Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2073 return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2076 protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2077 Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2078 TypedWriteTransaction<Configuration> writeConfigTxn) {
2079 String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2080 String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2082 Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2083 dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2084 if (optVpnInterface.isPresent()) {
2085 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2086 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2087 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2088 LogicalDatastoreType.OPERATIONAL, path);
2089 if (optAdjacencies.isPresent()) {
2090 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2092 if (adjacencies != null && !adjacencies.isEmpty()) {
2093 LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2094 for (Adjacency adjacency : adjacencies) {
2095 if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2096 String rd = adjacency.getVrfId();
2097 if (adj.getNextHopIpList() != null) {
2098 for (String nh : adj.getNextHopIpList()) {
2099 deleteExtraRouteFromCurrentAndImportingVpns(
2100 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2101 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2103 } else if (adj.isPhysNetworkFunc()) {
2104 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2105 adj.getIpAddress(), adj.getSubnetId());
2106 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2107 null, writeConfigTxn);
2114 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2115 dpnId, adj.getVrfId());
2117 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2118 + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2121 } catch (ReadFailedException e) {
2122 LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2123 adj.getIpAddress(), interfaceName, dpnId, vpnName);
2127 private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2128 String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2129 TypedWriteTransaction<Operational> writeOperTx) {
2130 LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2131 destination, nextHop, vpnName, intfName, rd);
2132 vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2133 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2134 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2135 String vpnRd = vpn.getVrfId();
2136 if (vpnRd != null) {
2137 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2138 destination, nextHop, vpnName, intfName, vpnRd);
2139 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2145 InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2146 return InstanceIdentifier.builder(NeutronRouterDpns.class)
2147 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2148 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2151 InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2152 return InstanceIdentifier.builder(NeutronRouterDpns.class)
2153 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2156 protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2157 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2158 if (vpnInterface == null) {
2161 List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2163 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2164 + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2167 for (Adjacency adj : adjs) {
2168 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2169 String primaryInterfaceIp = adj.getIpAddress();
2170 String macAddress = adj.getMacAddress();
2171 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2173 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2174 VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2176 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2177 .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2178 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2179 routerInt, label, writeConfigTxn);
2180 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2181 + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2184 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2185 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2186 interfaceName, primaryRd, vpnName);
2191 protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2192 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2193 Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2194 String rd = vpnUtil.getVpnRd(vpnName);
2196 List<Adjacency> adjsList = adjs.nonnullAdjacency();
2197 for (Adjacency adj : adjsList) {
2198 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2199 String primaryInterfaceIp = adj.getIpAddress();
2200 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2201 fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2202 LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2203 + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2207 LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2208 vpnInterface.getName(), rd);
2212 private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2213 final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2214 final String interfaceName = key.getName();
2215 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2216 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2217 addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2218 intefaceData.identifier, vpnName);
2221 private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2222 VpnInterface vpnInterface, String vpnName) {
2223 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2225 if (vpnInterfaces == null) {
2226 vpnInterfaces = new ConcurrentLinkedQueue<>();
2228 vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2229 unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2230 LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2231 vpnInterface.getName(), vpnName);
2234 public boolean isVpnInstanceReady(String vpnInstanceName) {
2235 String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2236 if (vpnRd == null) {
2239 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2241 return vpnInstanceOpDataEntry != null;
2244 public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2245 // FIXME: separate out to somehow?
2246 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2249 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2250 unprocessedVpnInterfaces.get(vpnInstanceName);
2251 if (vpnInterfaces != null) {
2252 while (!vpnInterfaces.isEmpty()) {
2253 UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2254 if (hasVpnInstanceCreatedSuccessfully) {
2255 processSavedInterface(savedInterface, vpnInstanceName);
2256 LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2257 savedInterface.vpnInterface.getName(), vpnInstanceName);
2259 LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2260 savedInterface.vpnInterface.getName(), vpnInstanceName);
2264 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2271 private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2272 VpnInterface vpnInterface) {
2273 // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2274 final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2275 final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2278 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2279 unprocessedVpnInterfaces.get(firstVpnName);
2280 if (vpnInterfaces != null) {
2281 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2282 LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2283 + "unprocessed list", vpnInterface.getName(), firstVpnName);
2286 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2293 public void vpnInstanceIsReady(String vpnInstanceName) {
2294 processSavedInterfaces(vpnInstanceName, true);
2297 public void vpnInstanceFailed(String vpnInstanceName) {
2298 processSavedInterfaces(vpnInstanceName, false);
2301 private static class UnprocessedVpnInterfaceData {
2302 InstanceIdentifier<VpnInterface> identifier;
2303 VpnInterface vpnInterface;
2305 UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2306 this.identifier = identifier;
2307 this.vpnInterface = vpnInterface;
2311 public int hashCode() {
2312 final int prime = 31;
2314 result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2315 result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2320 public boolean equals(Object obj) {
2327 if (getClass() != obj.getClass()) {
2330 UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2331 if (identifier == null) {
2332 if (other.identifier != null) {
2335 } else if (!identifier.equals(other.identifier)) {
2338 if (vpnInterface == null) {
2339 if (other.vpnInterface != null) {
2342 } else if (!vpnInterface.equals(other.vpnInterface)) {
2349 public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2350 String primaryRd = vpnUtil.getVpnRd(vpnName);
2351 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2352 if (vpnInstanceOpData == null) {
2355 List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2356 if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2359 LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2360 vpnToDpnLists.forEach(vpnToDpnList -> {
2361 if (vpnToDpnList.getVpnInterfaces() == null) {
2364 vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2366 InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2367 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2368 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2369 .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2370 if (!vpnInterfaceOptional.isPresent()) {
2373 List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2374 vpnInterface.getInterfaceName());
2375 if (configVpnAdjacencies == null) {
2376 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2379 List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2380 .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2381 // Due to insufficient rds, some of the extra route wont get processed when it is added.
2382 // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2383 // in operational DS. These unprocessed adjacencies will be handled below.
2384 // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2385 // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2386 configVpnAdjacencies.stream()
2387 .filter(adjacency -> operationVpnAdjacencies.stream()
2388 .noneMatch(operationalAdjacency ->
2389 Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2390 .forEach(adjacency -> {
2391 LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2392 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2394 // TODO Deal with sequencing — the config tx must only submitted
2395 // if the oper tx goes in
2396 if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2397 List<ListenableFuture<Void>> futures = new ArrayList<>();
2399 txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2400 //set of prefix used, as entry in prefix-to-interface datastore
2401 // is prerequisite for refresh Fib to avoid race condition leading
2402 // to missing remote next hop in bucket actions on bgp-vpn delete
2403 Set<String> prefixListForRefreshFib = new HashSet<>();
2404 ListenableFuture<Void> configTxFuture =
2405 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2406 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2407 primaryRd, adjacency,
2408 vpnInterfaceOptional.get().getDpnId(),
2409 operTx, confTx, confTx, prefixListForRefreshFib));
2410 Futures.addCallback(configTxFuture,
2411 new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2412 MoreExecutors.directExecutor());
2413 futures.add(configTxFuture);
2421 } catch (ReadFailedException e) {
2422 LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2423 vpnName, primaryRd);
2429 private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2430 private final String interfaceName;
2431 private final boolean add;
2432 private final String txnDestination;
2434 PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2435 this.interfaceName = interfaceName;
2437 this.txnDestination = transactionDest;
2441 public void onSuccess(Void voidObj) {
2443 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2444 interfaceName, txnDestination);
2446 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2451 public void onFailure(Throwable throwable) {
2453 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2454 interfaceName, txnDestination, throwable);
2456 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2457 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2462 private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2463 private final String primaryRd;
2464 private final Set<String> prefixListForRefreshFib;
2466 VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2467 this.primaryRd = primaryRd;
2468 this.prefixListForRefreshFib = prefixListForRefreshFib;
2472 public void onSuccess(Void voidObj) {
2473 prefixListForRefreshFib.forEach(prefix -> {
2474 fibManager.refreshVrfEntry(primaryRd, prefix);
2479 public void onFailure(Throwable throwable) {
2480 LOG.debug("write Tx config operation failed", throwable);