2 * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.vpnmanager;
10 import static java.util.Collections.emptyList;
11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
14 import com.google.common.base.Preconditions;
15 import com.google.common.collect.Iterators;
16 import com.google.common.util.concurrent.FutureCallback;
17 import com.google.common.util.concurrent.Futures;
18 import com.google.common.util.concurrent.ListenableFuture;
19 import com.google.common.util.concurrent.MoreExecutors;
20 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
21 import java.util.ArrayList;
22 import java.util.Collections;
23 import java.util.HashSet;
24 import java.util.List;
26 import java.util.Objects;
27 import java.util.Optional;
29 import java.util.concurrent.ConcurrentHashMap;
30 import java.util.concurrent.ConcurrentLinkedQueue;
31 import java.util.concurrent.ExecutionException;
32 import java.util.concurrent.locks.ReentrantLock;
33 import java.util.function.Consumer;
34 import java.util.function.Predicate;
35 import java.util.stream.Collectors;
36 import javax.annotation.PreDestroy;
37 import javax.inject.Inject;
38 import javax.inject.Singleton;
39 import org.eclipse.jdt.annotation.Nullable;
40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
41 import org.opendaylight.genius.infra.Datastore.Configuration;
42 import org.opendaylight.genius.infra.Datastore.Operational;
43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
45 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
46 import org.opendaylight.genius.infra.TypedWriteTransaction;
47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
48 import org.opendaylight.genius.mdsalutil.NWUtil;
49 import org.opendaylight.genius.mdsalutil.NwConstants;
50 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
51 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
52 import org.opendaylight.genius.utils.JvmGlobalLocks;
53 import org.opendaylight.infrautils.caches.CacheProvider;
54 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
55 import org.opendaylight.infrautils.utils.concurrent.Executors;
56 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
57 import org.opendaylight.mdsal.binding.api.DataBroker;
58 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
59 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
60 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
61 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
62 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
63 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
64 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
65 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
66 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
67 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
68 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
69 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
70 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
71 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces;
110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType;
112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyKey;
114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey;
116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
121 import org.opendaylight.yangtools.yang.common.Uint32;
122 import org.opendaylight.yangtools.yang.common.Uint64;
123 import org.slf4j.Logger;
124 import org.slf4j.LoggerFactory;
127 public class VpnInterfaceManager extends AbstractAsyncDataTreeChangeListener<VpnInterface> {
129 private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
130 private static final short DJC_MAX_RETRIES = 3;
132 private final DataBroker dataBroker;
133 private final ManagedNewTransactionRunner txRunner;
134 private final IBgpManager bgpManager;
135 private final IFibManager fibManager;
136 private final IMdsalApiManager mdsalManager;
137 private final IdManagerService idManager;
138 private final OdlInterfaceRpcService ifaceMgrRpcService;
139 private final VpnFootprintService vpnFootprintService;
140 private final IInterfaceManager interfaceManager;
141 private final IVpnManager vpnManager;
142 private final ArpResponderHandler arpResponderHandler;
143 private final JobCoordinator jobCoordinator;
144 private final VpnUtil vpnUtil;
146 private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
148 private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
149 new ConcurrentHashMap<>();
151 private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
154 public VpnInterfaceManager(final DataBroker dataBroker,
155 final IBgpManager bgpManager,
156 final IdManagerService idManager,
157 final IMdsalApiManager mdsalManager,
158 final IFibManager fibManager,
159 final OdlInterfaceRpcService ifaceMgrRpcService,
160 final VpnFootprintService vpnFootprintService,
161 final IInterfaceManager interfaceManager,
162 final IVpnManager vpnManager,
163 final ArpResponderHandler arpResponderHandler,
164 final JobCoordinator jobCoordinator,
165 final CacheProvider cacheProvider,
166 final VpnUtil vpnUtil) {
167 super(dataBroker, LogicalDatastoreType.CONFIGURATION,
168 InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class),
169 Executors.newListeningSingleThreadExecutor("VpnInterfaceManager", LOG));
171 this.dataBroker = dataBroker;
172 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
173 this.bgpManager = bgpManager;
174 this.idManager = idManager;
175 this.mdsalManager = mdsalManager;
176 this.fibManager = fibManager;
177 this.ifaceMgrRpcService = ifaceMgrRpcService;
178 this.vpnFootprintService = vpnFootprintService;
179 this.interfaceManager = interfaceManager;
180 this.vpnManager = vpnManager;
181 this.arpResponderHandler = arpResponderHandler;
182 this.jobCoordinator = jobCoordinator;
183 this.vpnUtil = vpnUtil;
185 vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
186 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
187 VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
191 public Runnable isNotifyTaskQueued(String intfName) {
192 return vpnIntfMap.remove(intfName);
195 public void start() {
196 LOG.info("{} start", getClass().getSimpleName());
201 public void close() {
203 vpnInstanceOpDataEntryCache.close();
204 Executors.shutdownAndAwaitTermination(getExecutorService());
208 public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
209 LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
210 LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
211 VpnHelper.getVpnInterfaceVpnInstanceNamesString(
212 new ArrayList<VpnInstanceNames>(vpnInterface.nonnullVpnInstanceNames().values())));
213 addVpnInterface(identifier, vpnInterface, null, null);
216 private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
217 final VpnInterface vpnInterface, String vpnName) {
218 // FIXME: separate this out somehow?
219 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
222 if (isVpnInstanceReady(vpnName)) {
225 addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
232 // TODO Clean up the exception handling
233 @SuppressWarnings("checkstyle:IllegalCatch")
234 private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
235 final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
236 for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) {
237 String vpnName = vpnInterfaceVpnInstance.getVpnName();
238 addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
242 private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
243 final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
244 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
245 final String interfaceName = key.getName();
247 if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
248 LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
249 vpnName, vpnInterface.getName());
252 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
253 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
254 List<Adjacency> copyOldAdjs = null;
255 if (oldAdjs != null) {
256 copyOldAdjs = new ArrayList<>();
257 copyOldAdjs.addAll(oldAdjs);
259 List<Adjacency> copyNewAdjs = null;
260 if (newAdjs != null) {
261 copyNewAdjs = new ArrayList<>();
262 copyNewAdjs.addAll(newAdjs);
264 addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
267 private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
268 final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
269 final @Nullable List<Adjacency> newAdjs,
270 final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
271 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
272 final String interfaceName = key.getName();
273 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
274 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
275 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
276 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
277 if (interfaceState != null) {
279 final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
280 final int ifIndex = interfaceState.getIfIndex();
281 jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
282 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
283 // (the inventory tx goes in last)
284 List<ListenableFuture<Void>> futures = new ArrayList<>();
285 //set of prefix used, as entry in prefix-to-interface datastore
286 // is prerequisite for refresh Fib to avoid race condition leading to
287 // missing remote next hop in bucket actions on bgp-vpn delete
288 Set<String> prefixListForRefreshFib = new HashSet<>();
289 ListenableFuture<Void> confFuture =
290 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
291 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
292 operTx -> futures.add(
293 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
295 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
297 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
298 processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
299 confTx, operTx, invTx, interfaceState, vpnName,
300 prefixListForRefreshFib);
301 if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
302 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
303 + " Update for swapping VPN {} case.", interfaceName, vpnName);
304 if (newAdjs != null) {
305 for (Adjacency adj : newAdjs) {
306 if (oldAdjs.contains(adj)) {
309 if (!isBgpVpnInternetVpn
310 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
311 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
312 primaryRd, adj, dpnId, operTx, confTx, invTx,
313 prefixListForRefreshFib);
318 for (Adjacency adj : oldAdjs) {
319 if (!isBgpVpnInternetVpn
320 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
321 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
327 Futures.addCallback(confFuture,
328 new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
329 MoreExecutors.directExecutor());
330 futures.add(confFuture);
331 Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
332 MoreExecutors.directExecutor());
333 LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
334 + " processed successfully", interfaceName, vpnName, dpnId);
337 } catch (NumberFormatException | IllegalStateException e) {
338 LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
339 + "interface {}. Interface addition on vpn {} failed", interfaceName,
343 } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
344 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
346 ListenableFuture<Void> future =
347 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
348 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
350 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
351 vpnName, vpnInterface.getDpnId());
353 LoggingFutures.addErrorLogging(future, LOG,
354 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
355 return Collections.singletonList(future);
358 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
359 + " is not available", interfaceName, vpnName);
362 LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
363 + " as vpn is pending delete", interfaceName, vpnName,
364 vpnInterface.getDpnId());
368 // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
369 // see comments below.
370 @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
371 protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
372 final int lportTag, boolean isInterfaceUp,
373 TypedWriteTransaction<Configuration> writeConfigTxn,
374 TypedWriteTransaction<Operational> writeOperTxn,
375 TypedReadWriteTransaction<Configuration> writeInvTxn,
376 Interface interfaceState, final String vpnName,
377 Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
378 final String interfaceName = vpnInterface.getName();
379 Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
381 VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
382 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
383 if (!isInterfaceUp) {
384 LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
385 interfaceName, dpId, vpnName);
386 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
387 if (VpnConstants.INVALID_ID.equals(vpnId)) {
388 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
389 + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
394 boolean waitForVpnInterfaceOpRemoval = false;
395 if (opVpnInterface != null) {
396 String opVpnName = opVpnInterface.getVpnInstanceName();
397 String primaryInterfaceIp = null;
398 if (Objects.equals(opVpnName, vpnName)) {
399 // Please check if the primary VRF Entry does not exist for VPNInterface
400 // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
402 // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
403 List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
405 LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
406 + " for this vpn interface could not be obtained", interfaceName, dpId,
410 for (Adjacency adj : adjs) {
411 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
412 primaryInterfaceIp = adj.getIpAddress();
416 if (primaryInterfaceIp == null) {
417 LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
418 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
422 // Get the rd of the vpn instance
423 VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
425 LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
426 + " bailing out from here.", interfaceName, dpId, vpnName);
429 waitForVpnInterfaceOpRemoval = true;
431 LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
432 + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
435 if (!waitForVpnInterfaceOpRemoval) {
436 // Add the VPNInterface and quit
437 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
438 null/*ipAddressSourceValuePair*/,
440 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
441 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
442 if (!isBgpVpnInternetVpn) {
443 vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
445 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
447 if (interfaceManager.isExternalInterface(interfaceName)) {
448 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
449 NwConstants.ADD_FLOW);
454 // FIB didn't get a chance yet to clean up this VPNInterface
455 // Let us give it a chance here !
456 LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
457 + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
459 Runnable notifyTask = new VpnNotifyTask();
460 synchronized (notifyTask) {
461 // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
462 // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
463 // published here while holding the lock on it so this path will hit the wait before notify can be
465 vpnIntfMap.put(interfaceName, notifyTask);
467 notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
468 } catch (InterruptedException e) {
473 vpnIntfMap.remove(interfaceName);
476 if (opVpnInterface != null) {
477 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
478 + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
480 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
483 // VPNInterface got removed, proceed with Add
484 LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
485 interfaceName, dpId, vpnName);
486 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
487 null/*ipAddressSourceValuePair*/,
489 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
490 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
491 if (!isBgpVpnInternetVpn) {
492 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
494 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
495 + " FIB to clean up", interfaceName, dpId, vpnName);
496 if (interfaceManager.isExternalInterface(interfaceName)) {
497 processExternalVpnInterface(interfaceName, vpnName, dpId,
498 lportTag, NwConstants.ADD_FLOW);
502 // Interface is retained in the DPN, but its Link Up.
503 // Advertise prefixes again for this interface to BGP
504 InstanceIdentifier<VpnInterface> identifier =
505 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
506 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
507 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
508 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
509 // Perform similar operation as interface add event for extraroutes.
510 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
511 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
512 LogicalDatastoreType.CONFIGURATION, path);
513 if (!optAdjacencies.isPresent()) {
514 LOG.trace("No config adjacencyKeyAdjacencyMap present for vpninterface {}", vpnInterface);
517 Map<AdjacencyKey, Adjacency> adjacencyKeyAdjacencyMap = optAdjacencies.get().nonnullAdjacency();
518 for (Adjacency adjacency : adjacencyKeyAdjacencyMap.values()) {
519 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
522 // if BGPVPN Internet, filter only IPv6 Adjacencies
523 if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
526 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
527 dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
529 } catch (InterruptedException | ExecutionException e) {
530 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
531 interfaceName, vpnName, primaryRd, dpId);
536 private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
537 int lportTag, int addOrRemove) {
540 // vpn instance of ext-net interface is the network-id
541 extNetworkId = new Uuid(vpnName);
542 } catch (IllegalArgumentException e) {
543 LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
544 + " on dpn {} failed", vpnName, interfaceName, dpId);
548 List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
549 if (routerIds == null || routerIds.isEmpty()) {
550 LOG.info("processExternalVpnInterface: No router is associated with {}."
551 + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
552 extNetworkId.getValue(), interfaceName, dpId, vpnName);
556 LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
557 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
558 for (Uuid routerId : routerIds) {
559 String routerName = routerId.getValue();
560 Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
561 if (Objects.equals(primarySwitch, dpId)) {
562 Routers router = vpnUtil.getExternalRouter(routerName);
563 if (router != null) {
564 if (addOrRemove == NwConstants.ADD_FLOW) {
565 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
566 VpnUtil.getIpsListFromExternalIps(new ArrayList<ExternalIps>(router
567 .nonnullExternalIps().values())), router.getExtGwMacAddress(),
568 dpId, interfaceName, lportTag);
570 vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
571 VpnUtil.getIpsListFromExternalIps(new ArrayList<ExternalIps>(router
572 .nonnullExternalIps().values())),
573 dpId, interfaceName, lportTag);
576 LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
577 + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
578 interfaceName, dpId, vpnName);
584 // TODO Clean up the exception handling
585 @SuppressWarnings("checkstyle:IllegalCatch")
586 private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
587 final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
588 String vpnName, String interfaceName) {
590 LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
591 interfaceName, dpnId, vpnName);
594 if (rd.equals(vpnName)) {
595 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
596 + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
600 LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
601 interfaceName, dpnId, vpnName, rd);
603 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
604 if (nextHopIp == null) {
605 LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
606 + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
613 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
614 Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
615 LogicalDatastoreType.OPERATIONAL, path);
616 if (adjacencies.isPresent()) {
617 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.get().getAdjacency();
618 if (nextHopsMap != null && !nextHopsMap.isEmpty()) {
619 LOG.debug("advertiseAdjacenciesForVpnToBgp: NextHops are {} for interface {} on dpn {} for vpn {}"
620 + " rd {}", nextHopsMap, interfaceName, dpnId, vpnName, rd);
621 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
622 Uint32 l3vni = vpnInstanceOpData.getL3vni();
623 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
624 ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
625 for (Adjacency nextHop : nextHopsMap.values()) {
626 if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
629 String gatewayMac = null;
630 Uint32 label = Uint32.ZERO;
631 if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
632 final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
633 vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
634 gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
637 label = nextHop.getLabel();
640 LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
641 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
642 bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
643 encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
645 LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
646 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
647 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
648 } catch (Exception e) {
649 LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
650 + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
651 interfaceName, dpnId, e);
656 } catch (InterruptedException | ExecutionException e) {
657 LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
658 + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
662 // TODO Clean up the exception handling
663 @SuppressWarnings("checkstyle:IllegalCatch")
664 private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
665 String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
666 TypedWriteTransaction<Operational> writeOperTx) {
668 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
669 String rd = vpnUtil.getVpnRd(interfaceName);
671 LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
672 interfaceName, vpnName);
675 if (rd.equals(vpnName)) {
677 "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
678 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
682 LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
684 Optional<AdjacenciesOp> adjacencies = Optional.empty();
686 adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
688 } catch (InterruptedException | ExecutionException e) {
689 LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
690 interfaceName, vpnName);
692 if (adjacencies.isPresent()) {
693 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.get().getAdjacency();
695 if (nextHopsMap != null && !nextHopsMap.isEmpty()) {
696 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
697 nextHopsMap, interfaceName, vpnName, rd);
698 for (Adjacency nextHop : nextHopsMap.values()) {
700 if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
701 LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
702 + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
703 interfaceName, vpnName);
704 bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
705 LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
706 + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
707 interfaceName, vpnName);
708 } else if (nextHop.getNextHopIpList() != null) {
709 // Perform similar operation as interface delete event for extraroutes.
710 String allocatedRd = nextHop.getVrfId();
711 for (String nh : nextHop.getNextHopIpList()) {
712 deleteExtraRouteFromCurrentAndImportingVpns(
713 vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
717 } catch (Exception e) {
718 LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
719 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
726 @SuppressWarnings("checkstyle:IllegalCatch")
727 protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
728 String primaryRd, String interfaceName, final Uint32 vpnId,
729 TypedWriteTransaction<Configuration> writeConfigTxn,
730 TypedWriteTransaction<Operational> writeOperTxn,
731 TypedReadWriteTransaction<Configuration> writeInvTxn,
732 Interface interfaceState, Set<String> prefixListForRefreshFib)
733 throws ExecutionException, InterruptedException {
734 InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
736 Optional<VpnInterface> vpnInteface = Optional.empty();
738 vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
739 LogicalDatastoreType.CONFIGURATION, identifier);
740 } catch (InterruptedException | ExecutionException e) {
741 LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
742 + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
744 Uuid intfnetworkUuid = null;
745 NetworkType networkType = null;
746 Long segmentationId = Long.valueOf(-1);
747 Adjacencies adjacencies = null;
748 if (vpnInteface.isPresent()) {
749 intfnetworkUuid = vpnInteface.get().getNetworkId();
750 networkType = vpnInteface.get().getNetworkType();
751 segmentationId = vpnInteface.get().getSegmentationId().toJava();
752 adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
753 if (adjacencies == null) {
754 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
755 null/*gwMac*/, null/*gatewayIp*/, writeOperTxn);
759 // Get the rd of the vpn instance
760 String nextHopIp = null;
761 String gatewayIp = null;
763 nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
764 } catch (Exception e) {
765 LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
766 + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
768 List<String> nhList = new ArrayList<>();
769 if (nextHopIp != null) {
770 nhList.add(nextHopIp);
771 LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
772 interfaceName, dpnId, vpnName, nhList);
774 Optional<String> gwMac = Optional.empty();
775 String vpnInterfaceSubnetGwMacAddress = null;
776 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
777 Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
778 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
779 VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
780 VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
781 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies != null ? adjacencies.getAdjacency()
782 : Collections.<AdjacencyKey, Adjacency>emptyMap();
783 List<Adjacency> value = new ArrayList<>();
784 for (Adjacency nextHop : nextHopsMap.values()) {
785 String rd = primaryRd;
786 String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
787 if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
788 && NWUtil.isIpv4Address(nexthopIpValue)) {
789 String prefix = nextHop.getIpAddress() == null ? "null" :
790 VpnUtil.getIpPrefix(nextHop.getIpAddress());
791 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
792 + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
795 if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
796 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
797 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
798 ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
799 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHopsMap {} "
800 + "on dpn {} for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
802 Prefixes prefixes = intfnetworkUuid != null
803 ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
804 segmentationId, prefixCue) :
805 VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
806 writeOperTxn.mergeParentStructureMerge(VpnUtil.getPrefixToInterfaceIdentifier(
807 vpnUtil.getVpnId(vpnName), prefix), prefixes);
808 final Uuid subnetId = nextHop.getSubnetId();
810 gatewayIp = nextHop.getSubnetGatewayIp();
811 if (gatewayIp == null) {
812 Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
813 if (gatewayIpOptional.isPresent()) {
814 gatewayIp = gatewayIpOptional.get();
818 if (gatewayIp != null) {
819 gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
820 if (gwMac.isPresent()) {
821 // A valid mac-address is available for this subnet-gateway-ip
822 // Use this for programming ARP_RESPONDER table here. And save this
823 // info into vpnInterface operational, so it can used in VrfEntryProcessor
824 // to populate L3_GW_MAC_TABLE there.
825 arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
826 gatewayIp, gwMac.get());
827 vpnInterfaceSubnetGwMacAddress = gwMac.get();
829 // A valid mac-address is not available for this subnet-gateway-ip
830 // Use the connected-mac-address to configure ARP_RESPONDER Table.
831 // Save this connected-mac-address as gateway-mac-address for the
832 // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
833 gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
834 if (gwMac.isPresent()) {
835 vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
836 NwConstants.ADD_FLOW, gwMac.get());
837 arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
838 gatewayIp, gwMac.get());
840 LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
841 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
843 subnetId, interfaceName, vpnName, gatewayIp);
847 LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
848 + "cannot create ARP responder flow for interface name {}, vpnName {}",
849 subnetId, interfaceName, vpnName);
850 gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
852 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHopsMap {} on dpn {}"
853 + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
855 //Extra route adjacency
856 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
857 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
858 // FIXME: separate this out somehow?
859 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
862 java.util.Optional<String> rdToAllocate = vpnUtil
863 .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
864 nextHop.getNextHopIpList().get(0), dpnId);
865 if (rdToAllocate.isPresent()) {
866 rd = rdToAllocate.get();
867 LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
870 LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
876 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
877 + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
878 interfaceName, dpnId);
880 // Please note that primary adjacency will use a subnet-gateway-mac-address that
881 // can be different from the gateway-mac-address within the VRFEntry as the
882 // gateway-mac-address is a superset.
883 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
884 L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
885 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
886 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
887 Adjacency operationalAdjacency = null;
889 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
890 } catch (NullPointerException e) {
891 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
892 input, e.getMessage());
895 if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
896 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
897 vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
900 value.add(operationalAdjacency);
903 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
904 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
905 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
907 L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
908 .setGatewayMac(gwMac.orElse(null)).setInterfaceName(interfaceName)
909 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
911 for (Adjacency nextHop : aug.nonnullAdjacency().values()) {
912 // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
913 if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
914 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
915 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
916 registeredPopulator.populateFib(input, writeConfigTxn);
921 private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
922 long lportTag, String gwMac, String gwIp,
923 TypedWriteTransaction<Operational> writeOperTxn) {
924 VpnInterfaceOpDataEntry opInterface =
925 VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
926 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
927 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
928 writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
929 LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
930 interfaceName, dpnId, vpnName);
933 // TODO Clean up the exception handling
934 @SuppressWarnings("checkstyle:IllegalCatch")
935 public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
936 StateTunnelList stateTunnelList,
937 TypedWriteTransaction<Configuration> writeConfigTxn,
938 TypedWriteTransaction<Operational> writeOperTxn) {
940 String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
941 Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
942 AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
943 Map<AdjacencyKey, Adjacency> keyAdjacencyMap =
944 adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency()
945 : Collections.<AdjacencyKey, Adjacency>emptyMap();
946 if (keyAdjacencyMap.isEmpty()) {
947 LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
948 vpnInterface, srcDpnId);
951 String prefix = null;
952 List<Adjacency> value = new ArrayList<>();
953 boolean isFibNextHopAddReqd = false;
954 String vpnName = vpnInterface.getVpnInstanceName();
955 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
956 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
957 LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
958 vpnInterface.getName(), vpnInterface.getDpnId(),
959 vpnInterface.getVpnInstanceName(), keyAdjacencyMap);
960 for (Adjacency adj : keyAdjacencyMap.values()) {
961 String rd = adj.getVrfId();
962 rd = rd != null ? rd : vpnName;
963 prefix = adj.getIpAddress();
964 Uint32 label = adj.getLabel();
965 List<String> nhList = Collections.singletonList(srcTepIp);
966 List<String> nextHopList = adj.getNextHopIpList();
967 // If TEP is added , update the nexthop of primary adjacency.
968 // Secondary adj nexthop is already pointing to primary adj IP address.
969 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
970 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
971 if (nextHopList != null && !nextHopList.isEmpty()) {
972 /* everything right already */
974 isFibNextHopAddReqd = true;
977 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
978 if (!vrfEntryOptional.isPresent()) {
981 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
982 if (!nhList.contains(srcTepIp)) {
983 nhList.add(srcTepIp);
984 isFibNextHopAddReqd = true;
989 if (isFibNextHopAddReqd) {
990 updateLabelMapper(label, nhList);
991 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
992 + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
993 vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
994 // Update the VRF entry with nextHop
995 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, true,
998 //Get the list of VPN's importing this route(prefix) .
999 // Then update the VRF entry with nhList
1000 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1001 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1002 String vpnRd = vpn.getVrfId();
1003 if (vpnRd != null) {
1004 fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, true,
1006 LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1007 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1008 vpnInterface.getName(), srcDpnId, vpnName,
1009 vpn.getVpnInstanceName(), vpnRd);
1012 // Advertise the prefix to BGP only for external vpn
1013 // since there is a nexthop change.
1015 if (!rd.equalsIgnoreCase(vpnName)) {
1016 bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1017 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1018 null /*gatewayMacAddress*/);
1020 LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1021 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1023 } catch (Exception ex) {
1024 LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1025 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1026 vpnInterface.getName(), srcDpnId, vpnName, ex);
1030 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1031 VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1032 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1033 .addAugmentation(AdjacenciesOp.class, aug).build();
1034 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1035 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1036 writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
1037 LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1038 vpnInterface.getName(), srcDpnId, vpnName);
1042 // TODO Clean up the exception handling
1043 @SuppressWarnings("checkstyle:IllegalCatch")
1044 public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1045 StateTunnelList stateTunnelList,
1046 TypedWriteTransaction<Configuration> writeConfigTxn,
1047 TypedWriteTransaction<Operational> writeOperTxn) {
1049 AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1050 List<Adjacency> adjList = adjacencies != null ? new ArrayList<Adjacency>(adjacencies
1051 .nonnullAdjacency().values()) : new ArrayList<>();
1052 String prefix = null;
1053 boolean isNextHopRemoveReqd = false;
1054 String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1055 Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1056 String vpnName = vpnInterface.getVpnInstanceName();
1057 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1058 String primaryRd = vpnUtil.getVpnRd(vpnName);
1059 if (adjList != null) {
1060 List<Adjacency> value = new ArrayList<>();
1061 LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1062 vpnInterface.getName(), vpnInterface.getDpnId(),
1063 vpnInterface.getVpnInstanceName(), adjList);
1064 for (Adjacency adj : adjList) {
1065 List<String> nhList = new ArrayList<>();
1066 String rd = adj.getVrfId();
1067 rd = rd != null ? rd : vpnName;
1068 prefix = adj.getIpAddress();
1069 List<String> nextHopList = adj.getNextHopIpList();
1070 Uint32 label = adj.getLabel();
1071 if (nextHopList != null && !nextHopList.isEmpty()) {
1072 isNextHopRemoveReqd = true;
1074 // If TEP is deleted , remove the nexthop from primary adjacency.
1075 // Secondary adj nexthop will continue to point to primary adj IP address.
1076 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1077 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1079 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1080 if (!vrfEntryOptional.isPresent()) {
1083 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1084 if (nhList.contains(srcTepIp)) {
1085 nhList.remove(srcTepIp);
1086 isNextHopRemoveReqd = true;
1091 if (isNextHopRemoveReqd) {
1092 updateLabelMapper(label, nhList);
1093 LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1094 + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1095 prefix, nhList, vpnName,
1096 vpnId, rd, vpnInterface.getName());
1097 // Update the VRF entry with removed nextHop
1098 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, false,
1101 //Get the list of VPN's importing this route(prefix) .
1102 // Then update the VRF entry with nhList
1103 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1104 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1105 String vpnRd = vpn.getVrfId();
1106 if (vpnRd != null) {
1107 fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, false,
1109 LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1110 + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1111 nhList, label, vpnInterface.getName(), srcDpnId,
1113 vpn.getVpnInstanceName(), vpnRd);
1117 // Withdraw prefix from BGP only for external vpn.
1119 if (!rd.equalsIgnoreCase(vpnName)) {
1120 bgpManager.withdrawPrefix(rd, prefix);
1122 LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1123 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1124 vpnInterface.getName(), srcDpnId,
1126 } catch (Exception ex) {
1127 LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1128 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1129 vpnInterface.getName(), srcDpnId, vpnName, ex);
1133 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1134 VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1135 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1136 .addAugmentation(AdjacenciesOp.class, aug).build();
1137 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1138 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1139 writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
1140 LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1141 vpnInterface.getName(), srcDpnId, vpnName);
1145 @SuppressWarnings("checkstyle:IllegalCatch")
1146 private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1147 List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1148 final VpnInstanceOpDataEntry vpnInstanceOpDataEntry;
1149 String vpnRd = vpnUtil.getVpnRd(vpnName);
1151 VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1152 if (opDataEntry == null) {
1153 LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}"
1154 + " when check for vpns exporting the routes", vpnName, vpnRd);
1155 return vpnsToExportRoute;
1157 vpnInstanceOpDataEntry = opDataEntry;
1158 } catch (Exception re) {
1159 LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}"
1160 + " to check for vpns exporting the routes", vpnName, vpnRd, re);
1161 return vpnsToExportRoute;
1163 Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1164 if (input.getVpnInstanceName() == null) {
1165 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {} without a name",
1169 return !input.getVpnInstanceName().equals(vpnName);
1172 Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1173 Iterable<String> commonRTs =
1174 VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1175 VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1176 return Iterators.size(commonRTs.iterator()) > 0;
1180 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1181 Collectors.toList());
1182 return vpnsToExportRoute;
1185 // TODO Clean up the exception handling
1186 @SuppressWarnings("checkstyle:IllegalCatch")
1187 void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1188 List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1189 for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1190 List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1191 if (vrfEntries != null) {
1192 LoggingFutures.addErrorLogging(
1193 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1194 for (VrfEntry vrfEntry : vrfEntries) {
1196 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1197 RouteOrigin.value(vrfEntry.getOrigin()))) {
1198 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1199 + " is not a controller managed non intervpn link route. Ignoring.",
1200 vpn.getVrfId(), vrfEntry.getDestPrefix());
1203 String prefix = vrfEntry.getDestPrefix();
1204 String gwMac = vrfEntry.getGatewayMacAddress();
1205 vrfEntry.nonnullRoutePaths().values().forEach(routePath -> {
1206 String nh = routePath.getNexthopAddress();
1207 Uint32 label = routePath.getLabel();
1208 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1209 vrfEntry.getOrigin()))) {
1211 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1212 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1213 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1214 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1215 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1216 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1219 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1220 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1221 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1222 SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1223 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1227 } catch (RuntimeException e) {
1228 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1229 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1230 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1233 }), LOG, "Error handing VPN exporting routes");
1235 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1236 vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1242 public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1243 LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1244 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1245 final String interfaceName = key.getName();
1246 for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) {
1247 String vpnName = vpnInterfaceVpnInstance.getVpnName();
1248 removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1252 private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1253 final VpnInterface vpnInterface, final String vpnName,
1254 final String interfaceName) {
1255 if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1256 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1257 ListenableFuture<Void> future =
1258 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1259 deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1260 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1262 LoggingFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1263 vpnInterface.getName(), vpnName);
1264 return Collections.singletonList(future);
1265 }, DJC_MAX_RETRIES);
1267 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1268 removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1272 @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1273 private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1274 final VpnInterface vpnInterface, final String vpnName,
1275 final String interfaceName, final Interface interfaceState) {
1276 LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1277 vpnName, vpnInterface.getDpnId());
1278 removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1279 jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1281 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1282 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1283 writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1284 writeOperTxn -> futures.add(
1285 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1286 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1287 interfaceName, vpnName);
1290 String gwMacAddress;
1291 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1292 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1293 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1295 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1296 LogicalDatastoreType.OPERATIONAL, interfaceId);
1297 } catch (InterruptedException | ExecutionException e) {
1298 LOG.error("remove: Failed to read data store for interface {} vpn {}",
1299 interfaceName, vpnName);
1302 if (interfaceState != null) {
1304 dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1305 } catch (NumberFormatException | IllegalStateException e) {
1306 LOG.error("remove: Unable to retrieve dpnId from interface operational"
1307 + " data store for interface {} on dpn {} for vpn {} Fetching"
1308 + " from vpn interface op data store. ", interfaceName,
1309 vpnInterface.getDpnId(), vpnName, e);
1312 ifIndex = interfaceState.getIfIndex();
1313 gwMacAddress = interfaceState.getPhysAddress().getValue();
1315 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1316 + " from vpn interface op.", interfaceName);
1317 if (optVpnInterface.isPresent()) {
1318 VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1319 dpId = vpnOpInterface.getDpnId();
1320 ifIndex = vpnOpInterface.getLportTag().intValue();
1321 gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1323 LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1324 + " as interfaceState and vpn interface op is not"
1325 + " available", interfaceName, vpnName);
1329 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1330 optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1331 writeConfigTxn, writeOperTxn, writeInvTxn);
1333 "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1335 interfaceName, vpnInterface.getDpnId(), vpnName);
1337 futures.add(configFuture);
1338 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1339 interfaceName, false, "Config"), MoreExecutors.directExecutor());
1341 }, DJC_MAX_RETRIES);
1344 protected void processVpnInterfaceDown(Uint64 dpId,
1345 String interfaceName,
1348 VpnInterfaceOpDataEntry vpnOpInterface,
1349 boolean isInterfaceStateDown,
1350 TypedWriteTransaction<Configuration> writeConfigTxn,
1351 TypedWriteTransaction<Operational> writeOperTxn,
1352 TypedReadWriteTransaction<Configuration> writeInvTxn)
1353 throws ExecutionException, InterruptedException {
1354 if (vpnOpInterface == null) {
1355 LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1356 + " as it is not available in operational data store", interfaceName, dpId);
1359 final String vpnName = vpnOpInterface.getVpnInstanceName();
1360 InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1361 interfaceName, vpnName);
1362 if (!isInterfaceStateDown) {
1363 final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1364 vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName, null);
1365 final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1366 removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1367 vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1368 if (interfaceManager.isExternalInterface(interfaceName)) {
1369 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1370 NwConstants.DEL_FLOW);
1372 if (!isBgpVpnInternetVpn) {
1373 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1375 LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1376 + " successful", interfaceName, dpId, vpnName);
1378 // Interface is retained in the DPN, but its Link Down.
1379 // Only withdraw the prefixes for this interface from BGP
1380 withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1384 private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1385 final String vpnName, final Uint32 vpnId, String gwMac,
1386 TypedWriteTransaction<Configuration> writeConfigTxn,
1387 TypedWriteTransaction<Operational> writeOperTxn,
1388 TypedReadWriteTransaction<Configuration> writeInvTxn)
1389 throws ExecutionException, InterruptedException {
1392 InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1393 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1394 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1395 SingleTransactionDataBroker.syncReadOptional(dataBroker,
1396 LogicalDatastoreType.OPERATIONAL, identifier);
1397 boolean isNonPrimaryAdjIp = false;
1398 String primaryRd = vpnUtil.getVpnRd(vpnName);
1399 LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1400 interfaceName, dpnId, vpnName, primaryRd);
1401 if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1402 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1403 + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1406 AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1408 if (adjacencies != null && adjacencies.getAdjacency() != null) {
1409 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.nonnullAdjacency();
1410 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1411 interfaceName, dpnId, vpnName, nextHopsMap);
1412 for (Adjacency nextHop : nextHopsMap.values()) {
1413 if (nextHop.isPhysNetworkFunc()) {
1414 LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1415 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1416 fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1417 null, null/*writeCfgTxn*/);
1419 String rd = nextHop.getVrfId();
1420 List<String> nhList;
1421 if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1422 nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1423 isNonPrimaryAdjIp = Boolean.TRUE;
1425 // This is a primary adjacency
1426 nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1428 removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1429 vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1430 interfaceName, writeInvTxn);
1431 isNonPrimaryAdjIp = Boolean.FALSE;
1433 if (!nhList.isEmpty()) {
1434 if (Objects.equals(primaryRd, vpnName)) {
1435 //this is an internal vpn - the rd is assigned to the vpn instance name;
1436 //remove from FIB directly
1437 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1438 interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1440 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1441 interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn);
1444 LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1445 + " interface {}", nextHop.getIpAddress(), rd,
1446 nextHop.getAdjacencyType().toString(), interfaceName);
1447 bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1448 fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1451 String ip = nextHop.getIpAddress().split("/")[0];
1452 LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1453 if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1454 vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1455 LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1456 + " for Interface {} ip {} on dpn {} for vpn {}",
1457 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1459 // Remove the MIP-IP from VpnPortIpToPort.
1460 if (isNonPrimaryAdjIp) {
1461 VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1462 if (persistedIp != null && persistedIp.isLearntIp()
1463 && persistedIp.getPortName().equals(interfaceName)) {
1464 VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1466 "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1467 + "from VpnPortipToPort",
1468 persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1471 VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1472 if (vpnPortipToPort != null) {
1473 VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1474 LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1475 + "Interface {} ip {} on dpn {} for vpn {}",
1476 vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1480 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1481 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1482 + " Removing it.", interfaceName, vpnName, dpnId);
1483 writeOperTxn.delete(identifier);
1485 } catch (InterruptedException | ExecutionException e) {
1486 LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1487 interfaceName, dpnId, vpnName);
1491 private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1492 String interfaceName, Uint64 dpnId,
1493 TypedWriteTransaction<Configuration> writeConfigTxn,
1494 TypedWriteTransaction<Operational> writeOperTx) {
1496 String primaryRd = vpnUtil.getVpnRd(vpnName);
1497 String prefix = nextHop.getIpAddress();
1498 String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1499 LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1500 nextHop, vpnName, interfaceName, dpnId);
1501 // FIXME: separate this out somehow?
1502 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1505 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1506 prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1507 //If extra-route is present behind at least one VM, then do not remove or update
1508 //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1509 //prefixtointerface DS
1512 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1517 LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1518 + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1519 vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1523 private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1524 Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp,
1525 TypedWriteTransaction<Configuration> writeConfigTxn,
1526 TypedWriteTransaction<Operational> writeOperTx) {
1527 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1528 vpnUtil.getVpnsImportingMyRoute(vpnName);
1529 nhList.forEach((nh) -> {
1530 //IRT: remove routes from other vpns importing it
1531 if (isNonPrimaryAdjIp) {
1532 removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1534 vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1535 nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1537 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1538 String vpnRd = vpn.getVrfId();
1539 if (vpnRd != null) {
1540 fibManager.removeOrUpdateFibEntry(vpnRd,
1541 nextHop.getIpAddress(), nh, writeConfigTxn);
1542 LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1543 + " prefix {} nextHop {} from VPN {} parentVpn {}"
1544 + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1545 vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1551 @SuppressWarnings("checkstyle:IllegalCatch")
1552 private void removeLearntPrefixFromBGP(String rd, String prefix, String nextHop,
1553 TypedWriteTransaction<Configuration> writeConfigTxn) {
1555 if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) {
1556 LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this"
1557 + " operation", prefix, nextHop, rd);
1560 LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}",
1561 rd, prefix, nextHop);
1562 fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn);
1563 bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1564 LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}",
1565 rd, prefix, nextHop);
1566 } catch (Exception e) {
1567 LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e);
1571 private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1572 int lportTag, String gwMac, String gwIp, String interfaceName,
1573 TypedReadWriteTransaction<Configuration> writeInvTxn)
1574 throws ExecutionException, InterruptedException {
1575 final Uuid subnetId = nextHop.getSubnetId();
1576 if (nextHop.getSubnetGatewayMacAddress() == null) {
1577 // A valid mac-address was not available for this subnet-gateway-ip
1578 // So a connected-mac-address was used for this subnet and we need
1579 // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1580 vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1582 arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1586 private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1587 String interfaceName) {
1588 // This is either an extra-route (or) a learned IP via subnet-route
1589 List<String> nhList = null;
1590 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1591 if (nextHopIp == null || nextHopIp.isEmpty()) {
1592 LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1593 + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1594 + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1596 nhList = emptyList();
1598 nhList = Collections.singletonList(nextHopIp);
1603 private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1604 VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1605 //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1606 // else use connected interface
1607 if (gwPort != null && gwPort.isSubnetIp()) {
1608 LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1609 gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1610 return Optional.of(gwPort.getMacAddress());
1612 return Optional.empty();
1616 public void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1617 final VpnInterface update) {
1618 LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1619 LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1620 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1621 if (original.equals(update)) {
1622 LOG.info("update: original {} update {} are same. No update required.", original, update);
1625 final String vpnInterfaceName = update.getName();
1626 final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1627 LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1628 //handles switching between <internal VPN - external VPN>
1629 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1630 List<ListenableFuture<Void>> futures = new ArrayList<>();
1631 if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1632 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1633 + "to newVpn(s) {}",
1634 original.getName(), dpnId,
1635 VpnHelper.getVpnInterfaceVpnInstanceNamesString(
1636 new ArrayList<VpnInstanceNames>(original.nonnullVpnInstanceNames().values())),
1637 VpnHelper.getVpnInterfaceVpnInstanceNamesString(
1638 new ArrayList<VpnInstanceNames>(update.nonnullVpnInstanceNames().values())));
1641 updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1646 private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1647 VpnInterface original, VpnInterface update,
1648 List<ListenableFuture<Void>> futures) {
1649 boolean isVpnInstanceUpdate = false;
1650 final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1651 final String interfaceName = key.getName();
1652 List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1653 List<String> oldVpnListCopy = new ArrayList<>();
1654 oldVpnListCopy.addAll(oldVpnList);
1655 List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1656 List<String> newVpnListCopy = new ArrayList<>();
1657 newVpnListCopy.addAll(newVpnList);
1659 oldVpnList.removeAll(newVpnList);
1660 newVpnList.removeAll(oldVpnListCopy);
1661 //This block will execute only on if there is a change in the VPN Instance.
1662 if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1664 * Internet BGP-VPN Instance update with single router:
1665 * ====================================================
1666 * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1667 * 1st VPN Instance : router VPN or external BGP-VPN.
1668 * 2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1670 * VPN Instance UPDATE:
1671 * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1672 * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1674 * External BGP-VPN Instance update with single router:
1675 * ====================================================
1676 * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1678 * Updated VPN Instance will be always either internal router VPN to
1679 * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1681 * VPN Instance UPDATE:
1682 * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1683 * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1685 * Dual Router VPN Instance Update:
1686 * ================================
1687 * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1689 * 1st VPN Instance : router VPN or external BGP-VPN-1.
1690 * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1691 * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1693 * Dual Router --> Associated with common external BGP-VPN Instance.
1694 * 1st router and 2nd router are getting associated with single External BGP-VPN
1695 * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1696 * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1697 * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1701 isVpnInstanceUpdate = true;
1702 if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1703 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1704 && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1705 //Identify the external BGP-VPN Instance and pass that value as newVpnList
1706 List<String> externalBgpVpnList = new ArrayList<>();
1707 for (String newVpnName : newVpnListCopy) {
1708 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1709 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1710 if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1711 .BgpvpnType.BGPVPNExternal) {
1712 externalBgpVpnList.add(newVpnName);
1716 //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1717 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1718 externalBgpVpnList, oldVpnListCopy, futures);
1720 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1721 && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1722 //Identify the router VPN Instance and pass that value as oldVpnList
1723 List<String> routerVpnList = new ArrayList<>();
1724 for (String newVpnName : newVpnListCopy) {
1725 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1726 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1727 if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1729 routerVpnList.add(newVpnName);
1733 //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1734 updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1735 newVpnList, oldVpnListCopy, futures);
1738 //Handle remaining use cases.
1739 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1740 oldVpnListCopy, futures);
1743 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1744 oldVpnListCopy, futures);
1747 return isVpnInstanceUpdate;
1750 private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1751 VpnInterface original, VpnInterface update, List<String> oldVpnList,
1752 List<String> newVpnList, List<String> oldVpnListCopy,
1753 List<ListenableFuture<Void>> futures) {
1754 final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1755 final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1756 ? new ArrayList<Adjacency>(origAdjs.getAdjacency().values()) : new ArrayList<>();
1757 final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1758 final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1759 ? new ArrayList<Adjacency>(updateAdjs.getAdjacency().values()) : new ArrayList<>();
1761 boolean isOldVpnRemoveCallExecuted = false;
1762 for (String oldVpnName : oldVpnList) {
1763 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1764 + "remove from vpnName {} ", interfaceName, oldVpnName);
1765 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1766 LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1767 + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1769 isOldVpnRemoveCallExecuted = true;
1771 //Wait for previous interface bindings to be removed
1772 if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1775 } catch (InterruptedException e) {
1776 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1779 for (String newVpnName : newVpnList) {
1780 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1781 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1782 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1783 + "onto vpnName {} ", interfaceName, newVpnName);
1784 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1785 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1786 + "from oldVpn(s) {} to newVpn {} ",
1787 interfaceName, oldVpnListCopy, newVpnName);
1788 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1790 * In Dual stack network, first V4 subnet only attached to router and router is associated
1791 * with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1792 * Later V6 subnet is added to router, at this point existing VPN interface will get updated
1793 * with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1794 * is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1795 * BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1796 * router VPN as well as Internet BGP-VPN.
1798 * addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1799 * updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1800 * update in existing router VPN instance.
1802 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1803 LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1804 + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1805 updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1808 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1809 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1810 interfaceName, oldVpnListCopy, newVpnName);
1815 // TODO Clean up the exception handling
1816 @SuppressWarnings("checkstyle:IllegalCatch")
1817 private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1818 String vpnInterfaceName,
1819 List<ListenableFuture<Void>> futures) {
1820 final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1821 final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1822 != null ? new ArrayList<Adjacency>(origAdjs.getAdjacency().values()) : new ArrayList<>();
1823 final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1824 final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1825 != null ? new ArrayList<Adjacency>(updateAdjs.getAdjacency().values()) : new ArrayList<>();
1827 final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1828 for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames().values()) {
1829 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1830 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1831 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1832 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1833 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1834 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1835 //set of prefix used as entry in prefix-to-interface datastore
1836 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1837 // in bucket actions on bgp-vpn delete
1838 Set<String> prefixListForRefreshFib = new HashSet<>();
1839 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1840 confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1842 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1843 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1844 LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1845 update.getName(), newVpnName);
1846 //handle both addition and removal of adjacencies
1847 // currently, new adjacency may be an extra route
1848 boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1849 if (!oldAdjs.equals(newAdjs)) {
1850 for (Adjacency adj : copyNewAdjs) {
1851 if (copyOldAdjs.contains(adj)) {
1852 copyOldAdjs.remove(adj);
1854 // add new adjacency
1855 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1857 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1858 dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1859 } catch (RuntimeException e) {
1860 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1861 + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1864 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1865 + " added to vpn interface {} on vpn {} dpnId {}",
1866 adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1867 adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1870 for (Adjacency adj : copyOldAdjs) {
1871 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1872 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1873 && !adj.isPhysNetworkFunc()) {
1874 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1877 String vpnRd = vpnUtil.getVpnRd(newVpnName);
1878 LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1879 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1881 fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1882 if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1883 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1886 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1890 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1891 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1892 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1896 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1897 MoreExecutors.directExecutor());
1898 futures.add(configTxFuture);
1899 for (ListenableFuture<Void> future : futures) {
1900 LoggingFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1901 update.getName(), update.getVpnInstanceNames());
1904 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1905 + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1911 private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1912 final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1914 // FIXME: separate this out somehow?
1915 final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1918 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1919 .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1920 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1921 LogicalDatastoreType.OPERATIONAL, lriIid);
1922 if (opResult.isPresent()) {
1923 LabelRouteInfo labelRouteInfo =
1924 new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1925 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1926 labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1928 LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1930 } catch (InterruptedException | ExecutionException e) {
1931 LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1933 } catch (TransactionCommitFailedException e) {
1934 LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1941 public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1942 SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1944 RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1945 VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1946 .addAugmentation(SubnetRoute.class, route).build();
1947 List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1948 InstanceIdentifierBuilder<VrfTables> idBuilder =
1949 InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1950 InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1951 VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1952 if (writeConfigTxn != null) {
1953 writeConfigTxn.mergeParentStructureMerge(vrfTableId, vrfTableNew);
1955 vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1957 LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1958 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1961 protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1962 Adjacency adj, Uint64 dpnId,
1963 TypedWriteTransaction<Operational> writeOperTxn,
1964 TypedWriteTransaction<Configuration> writeConfigTxn,
1965 TypedReadWriteTransaction<Configuration> writeInvTxn,
1966 Set<String> prefixListForRefreshFib)
1967 throws ExecutionException, InterruptedException {
1968 String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1969 String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1971 Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1972 .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1973 if (optVpnInterface.isPresent()) {
1974 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1975 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1976 String vpnName = currVpnIntf.getVpnInstanceName();
1977 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1978 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1979 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1980 LogicalDatastoreType.OPERATIONAL, adjPath);
1981 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1982 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1983 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1984 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1985 List<Adjacency> adjacencies = new ArrayList<>();
1986 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1987 adjacencies.addAll(optAdjacencies.get().getAdjacency().values());
1989 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1990 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1991 .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1992 Adjacency operationalAdjacency = null;
1993 //Handling dual stack neutron port primary adjacency
1994 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1995 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1996 currVpnIntf.getName(), vpnName);
1997 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1998 currVpnIntf.getName());
1999 if (interfaceState != null) {
2000 processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
2001 currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
2002 prefixListForRefreshFib);
2005 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
2006 && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
2007 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
2008 : RouteOrigin.STATIC;
2009 String nh = adj.getNextHopIpList().get(0);
2010 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
2011 // FIXME: separate out to somehow?
2012 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
2015 java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
2016 vpnId, null, prefix, vpnName, nh, dpnId);
2017 if (rdToAllocate.isPresent()) {
2018 input.setRd(rdToAllocate.get());
2019 operationalAdjacency = populator.createOperationalAdjacency(input);
2020 int label = operationalAdjacency.getLabel().intValue();
2021 vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
2022 currVpnIntf.getVpnInstanceName(), l3vni, origin,
2023 currVpnIntf.getName(), operationalAdjacency, encapType,
2024 prefixListForRefreshFib, writeConfigTxn);
2025 LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
2026 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
2027 vpnName, label, currVpnIntf.getName(), dpnId);
2029 LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2033 // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2034 // Keeping the MPLS check for now.
2035 if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2036 final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2037 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2038 vpnUtil.getVpnsImportingMyRoute(vpnName);
2039 vpnsToImportRoute.forEach(vpn -> {
2040 if (vpn.getVrfId() != null) {
2041 vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2043 vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2045 rds -> vpnManager.addExtraRoute(
2046 vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2047 nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2048 RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2049 encapType, prefixListForRefreshFib, writeConfigTxn));
2056 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2057 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2058 currVpnIntf.getName(), vpnName);
2060 InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2061 .getVpnInterfaceIdentifier(currVpnIntf.getName());
2062 Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2063 LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2064 Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2065 Prefixes.PrefixCue.PhysNetFunc);
2066 if (vpnIntefaceConfig.isPresent()) {
2067 pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2068 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2069 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2072 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2074 writeOperTxn.mergeParentStructureMerge(
2075 VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2076 prefix), pnfPrefix);
2078 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2079 adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2080 Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2081 RouteOrigin.LOCAL, writeConfigTxn);
2083 input.setRd(adj.getVrfId());
2085 if (operationalAdjacency == null) {
2086 operationalAdjacency = populator.createOperationalAdjacency(input);
2088 adjacencies.add(operationalAdjacency);
2089 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2090 VpnInterfaceOpDataEntry newVpnIntf =
2091 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2092 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2093 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2094 writeOperTxn.mergeParentStructureMerge(identifier, newVpnIntf);
2096 } catch (InterruptedException | ExecutionException e) {
2097 LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2098 + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2103 private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2104 Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2105 return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2108 protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2109 Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2110 TypedWriteTransaction<Configuration> writeConfigTxn) {
2111 String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2112 String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2114 Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2115 dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2116 if (optVpnInterface.isPresent()) {
2117 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2118 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2119 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2120 LogicalDatastoreType.OPERATIONAL, path);
2121 if (optAdjacencies.isPresent()) {
2122 Map<AdjacencyKey, Adjacency> keyAdjacencyMap = optAdjacencies.get().getAdjacency();
2124 if (keyAdjacencyMap != null && !keyAdjacencyMap.isEmpty()) {
2125 LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", keyAdjacencyMap);
2126 for (Adjacency adjacency : keyAdjacencyMap.values()) {
2127 if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2128 String rd = adjacency.getVrfId();
2129 if (adj.getNextHopIpList() != null) {
2130 for (String nh : adj.getNextHopIpList()) {
2131 deleteExtraRouteFromCurrentAndImportingVpns(
2132 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2133 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2135 } else if (adj.isPhysNetworkFunc()) {
2136 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2137 adj.getIpAddress(), adj.getSubnetId());
2138 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2139 null, writeConfigTxn);
2146 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2147 dpnId, adj.getVrfId());
2149 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2150 + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2153 } catch (InterruptedException | ExecutionException e) {
2154 LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2155 adj.getIpAddress(), interfaceName, dpnId, vpnName);
2159 private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2160 String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2161 TypedWriteTransaction<Operational> writeOperTx) {
2162 LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2163 destination, nextHop, vpnName, intfName, rd);
2164 vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2165 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2166 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2167 String vpnRd = vpn.getVrfId();
2168 if (vpnRd != null) {
2169 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2170 destination, nextHop, vpnName, intfName, vpnRd);
2171 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2177 InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2178 return InstanceIdentifier.builder(NeutronRouterDpns.class)
2179 .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2180 .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2183 InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2184 return InstanceIdentifier.builder(NeutronRouterDpns.class)
2185 .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2188 protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2189 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2190 if (vpnInterface == null) {
2193 List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2195 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2196 + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2199 for (Adjacency adj : adjs) {
2200 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2201 String primaryInterfaceIp = adj.getIpAddress();
2202 String macAddress = adj.getMacAddress();
2203 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2205 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2206 VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2207 if (label.longValue() == VpnConstants.INVALID_LABEL) {
2209 "createFibEntryForRouterInterface: Unable to retrieve label for vpn pool {}, "
2210 + "vpninterface {}, vpn {}, rd {}",
2211 VpnConstants.VPN_IDPOOL_NAME, interfaceName, vpnName, primaryRd);
2214 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2215 .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2216 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2217 routerInt, label, writeConfigTxn);
2218 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2219 + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2222 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2223 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2224 interfaceName, primaryRd, vpnName);
2229 protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2230 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2231 Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2232 String rd = vpnUtil.getVpnRd(vpnName);
2234 Map<AdjacencyKey, Adjacency> keyAdjacencyMap = adjs.nonnullAdjacency();
2235 for (Adjacency adj : keyAdjacencyMap.values()) {
2236 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2237 String primaryInterfaceIp = adj.getIpAddress();
2238 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2239 fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2240 LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2241 + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2245 LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2246 vpnInterface.getName(), rd);
2250 private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2251 final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2252 final String interfaceName = key.getName();
2253 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2254 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2255 addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2256 intefaceData.identifier, vpnName);
2259 private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2260 VpnInterface vpnInterface, String vpnName) {
2261 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2263 if (vpnInterfaces == null) {
2264 vpnInterfaces = new ConcurrentLinkedQueue<>();
2266 vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2267 unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2268 LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2269 vpnInterface.getName(), vpnName);
2272 public boolean isVpnInstanceReady(String vpnInstanceName) {
2273 String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2274 if (vpnRd == null) {
2277 VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2279 return vpnInstanceOpDataEntry != null;
2282 public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2283 // FIXME: separate out to somehow?
2284 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2287 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2288 unprocessedVpnInterfaces.get(vpnInstanceName);
2289 if (vpnInterfaces != null) {
2290 while (!vpnInterfaces.isEmpty()) {
2291 UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2292 if (hasVpnInstanceCreatedSuccessfully) {
2293 processSavedInterface(savedInterface, vpnInstanceName);
2294 LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2295 savedInterface.vpnInterface.getName(), vpnInstanceName);
2297 LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2298 savedInterface.vpnInterface.getName(), vpnInstanceName);
2302 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2309 private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2310 VpnInterface vpnInterface) {
2311 // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2312 final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2313 final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2316 ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2317 unprocessedVpnInterfaces.get(firstVpnName);
2318 if (vpnInterfaces != null) {
2319 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2320 LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2321 + "unprocessed list", vpnInterface.getName(), firstVpnName);
2324 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2331 public void vpnInstanceIsReady(String vpnInstanceName) {
2332 processSavedInterfaces(vpnInstanceName, true);
2335 public void vpnInstanceFailed(String vpnInstanceName) {
2336 processSavedInterfaces(vpnInstanceName, false);
2339 private static class UnprocessedVpnInterfaceData {
2340 InstanceIdentifier<VpnInterface> identifier;
2341 VpnInterface vpnInterface;
2343 UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2344 this.identifier = identifier;
2345 this.vpnInterface = vpnInterface;
2349 public int hashCode() {
2350 final int prime = 31;
2352 result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2353 result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2358 public boolean equals(Object obj) {
2365 if (getClass() != obj.getClass()) {
2368 UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2369 if (identifier == null) {
2370 if (other.identifier != null) {
2373 } else if (!identifier.equals(other.identifier)) {
2376 if (vpnInterface == null) {
2377 if (other.vpnInterface != null) {
2380 } else if (!vpnInterface.equals(other.vpnInterface)) {
2387 public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2388 String primaryRd = vpnUtil.getVpnRd(vpnName);
2389 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2390 if (vpnInstanceOpData == null || vpnInstanceOpData.getVpnToDpnList() == null) {
2393 List<VpnToDpnList> vpnToDpnLists = new ArrayList<VpnToDpnList>(vpnInstanceOpData.getVpnToDpnList().values());
2394 if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2397 LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2398 vpnToDpnLists.forEach(vpnToDpnList -> {
2399 if (vpnToDpnList.getVpnInterfaces() == null) {
2402 vpnToDpnList.nonnullVpnInterfaces().values().forEach(vpnInterface -> {
2404 InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2405 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2406 Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2407 .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2408 if (!vpnInterfaceOptional.isPresent()) {
2411 List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2412 vpnInterface.getInterfaceName());
2413 if (configVpnAdjacencies == null) {
2414 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2417 List<Adjacency> operationVpnAdjacencies = new ArrayList<Adjacency>(vpnInterfaceOptional.get()
2418 .augmentation(AdjacenciesOp.class).nonnullAdjacency().values());
2419 // Due to insufficient rds, some of the extra route wont get processed when it is added.
2420 // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2421 // in operational DS. These unprocessed adjacencies will be handled below.
2422 // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2423 // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2424 configVpnAdjacencies.stream()
2425 .filter(adjacency -> operationVpnAdjacencies.stream()
2426 .noneMatch(operationalAdjacency ->
2427 Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2428 .forEach(adjacency -> {
2429 LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2430 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2432 // TODO Deal with sequencing — the config tx must only submitted
2433 // if the oper tx goes in
2434 if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2435 List<ListenableFuture<Void>> futures = new ArrayList<>();
2437 txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2438 //set of prefix used, as entry in prefix-to-interface datastore
2439 // is prerequisite for refresh Fib to avoid race condition leading
2440 // to missing remote next hop in bucket actions on bgp-vpn delete
2441 Set<String> prefixListForRefreshFib = new HashSet<>();
2442 ListenableFuture<Void> configTxFuture =
2443 txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2444 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2445 primaryRd, adjacency,
2446 vpnInterfaceOptional.get().getDpnId(),
2447 operTx, confTx, confTx, prefixListForRefreshFib));
2448 Futures.addCallback(configTxFuture,
2449 new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2450 MoreExecutors.directExecutor());
2451 futures.add(configTxFuture);
2459 } catch (InterruptedException | ExecutionException e) {
2460 LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2461 vpnName, primaryRd);
2467 private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2468 private final String interfaceName;
2469 private final boolean add;
2470 private final String txnDestination;
2472 PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2473 this.interfaceName = interfaceName;
2475 this.txnDestination = transactionDest;
2479 public void onSuccess(Void voidObj) {
2481 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2482 interfaceName, txnDestination);
2484 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2489 public void onFailure(Throwable throwable) {
2491 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2492 interfaceName, txnDestination, throwable);
2494 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2495 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2500 private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2501 private final String primaryRd;
2502 private final Set<String> prefixListForRefreshFib;
2504 VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2505 this.primaryRd = primaryRd;
2506 this.prefixListForRefreshFib = prefixListForRefreshFib;
2510 public void onSuccess(Void voidObj) {
2511 prefixListForRefreshFib.forEach(prefix -> {
2512 fibManager.refreshVrfEntry(primaryRd, prefix);
2517 public void onFailure(Throwable throwable) {
2518 LOG.debug("write Tx config operation failed", throwable);