vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static java.util.Objects.requireNonNull;
  12 import static org.opendaylight.mdsal.binding.util.Datastore.CONFIGURATION;
  13 import static org.opendaylight.mdsal.binding.util.Datastore.OPERATIONAL;
  14
  15 import com.google.common.collect.Iterators;
  16 import com.google.common.util.concurrent.FutureCallback;
  17 import com.google.common.util.concurrent.Futures;
  18 import com.google.common.util.concurrent.ListenableFuture;
  19 import com.google.common.util.concurrent.MoreExecutors;
  20 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Optional;
  28 import java.util.Set;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.locks.ReentrantLock;
  33 import java.util.function.Consumer;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.Nullable;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.mdsalutil.NWUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.utils.JvmGlobalLocks;
  47 import org.opendaylight.infrautils.caches.CacheProvider;
  48 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  49 import org.opendaylight.infrautils.utils.concurrent.Executors;
  50 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  51 import org.opendaylight.mdsal.binding.api.DataBroker;
  52 import org.opendaylight.mdsal.binding.util.Datastore.Configuration;
  53 import org.opendaylight.mdsal.binding.util.Datastore.Operational;
  54 import org.opendaylight.mdsal.binding.util.ManagedNewTransactionRunner;
  55 import org.opendaylight.mdsal.binding.util.ManagedNewTransactionRunnerImpl;
  56 import org.opendaylight.mdsal.binding.util.TypedReadWriteTransaction;
  57 import org.opendaylight.mdsal.binding.util.TypedWriteTransaction;
  58 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  59 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  60 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  61 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  62 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  63 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  64 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  65 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  66 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  67 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  68 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  69 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  70 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  71 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 118 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 120 import org.opendaylight.yangtools.yang.common.Uint32;
 121 import org.opendaylight.yangtools.yang.common.Uint64;
 122 import org.slf4j.Logger;
 123 import org.slf4j.LoggerFactory;
 124
 125 @Singleton
 126 public class VpnInterfaceManager extends AbstractAsyncDataTreeChangeListener<VpnInterface> {
 127
 128     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 129     private static final short DJC_MAX_RETRIES = 3;
 130
 131     private final DataBroker dataBroker;
 132     private final ManagedNewTransactionRunner txRunner;
 133     private final IBgpManager bgpManager;
 134     private final IFibManager fibManager;
 135     private final IMdsalApiManager mdsalManager;
 136     private final IdManagerService idManager;
 137     private final OdlInterfaceRpcService ifaceMgrRpcService;
 138     private final VpnFootprintService vpnFootprintService;
 139     private final IInterfaceManager interfaceManager;
 140     private final IVpnManager vpnManager;
 141     private final ArpResponderHandler arpResponderHandler;
 142     private final JobCoordinator jobCoordinator;
 143     private final VpnUtil vpnUtil;
 144
 145     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 146
 147     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 148             new ConcurrentHashMap<>();
 149
 150     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 151
 152     @Inject
 153     public VpnInterfaceManager(final DataBroker dataBroker,
 154                                final IBgpManager bgpManager,
 155                                final IdManagerService idManager,
 156                                final IMdsalApiManager mdsalManager,
 157                                final IFibManager fibManager,
 158                                final OdlInterfaceRpcService ifaceMgrRpcService,
 159                                final VpnFootprintService vpnFootprintService,
 160                                final IInterfaceManager interfaceManager,
 161                                final IVpnManager vpnManager,
 162                                final ArpResponderHandler arpResponderHandler,
 163                                final JobCoordinator jobCoordinator,
 164                                final CacheProvider cacheProvider,
 165                                final VpnUtil vpnUtil) {
 166         super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 167                 InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class),
 168                 Executors.newListeningSingleThreadExecutor("VpnInterfaceManager", LOG));
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187         start();
 188     }
 189
 190     public Runnable isNotifyTaskQueued(String intfName) {
 191         return vpnIntfMap.remove(intfName);
 192     }
 193
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196     }
 197
 198     @Override
 199     @PreDestroy
 200     public void close() {
 201         super.close();
 202         vpnInstanceOpDataEntryCache.close();
 203         Executors.shutdownAndAwaitTermination(getExecutorService());
 204     }
 205
 206     @Override
 207     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 208         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 209         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 210                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(
 211                         new ArrayList<>(vpnInterface.nonnullVpnInstanceNames().values())));
 212         addVpnInterface(identifier, vpnInterface, null, null);
 213     }
 214
 215     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 216                           final VpnInterface vpnInterface, String vpnName) {
 217         // FIXME: separate this out somehow?
 218         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 219         lock.lock();
 220         try {
 221             if (isVpnInstanceReady(vpnName)) {
 222                 return true;
 223             }
 224             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 225             return false;
 226         } finally {
 227             lock.unlock();
 228         }
 229     }
 230
 231     // TODO Clean up the exception handling
 232     @SuppressWarnings("checkstyle:IllegalCatch")
 233     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 234                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 235         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) {
 236             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 237             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 238         }
 239     }
 240
 241     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 242                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 243         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 244         final String interfaceName = key.getName();
 245
 246         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 247             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 248                   vpnName, vpnInterface.getName());
 249             return;
 250         }
 251         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 252              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 253         List<Adjacency> copyOldAdjs = null;
 254         if (oldAdjs != null) {
 255             copyOldAdjs = new ArrayList<>();
 256             copyOldAdjs.addAll(oldAdjs);
 257         }
 258         List<Adjacency> copyNewAdjs = null;
 259         if (newAdjs != null) {
 260             copyNewAdjs = new ArrayList<>();
 261             copyNewAdjs.addAll(newAdjs);
 262         }
 263         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 264     }
 265
 266     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 267                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 268                     final @Nullable List<Adjacency> newAdjs,
 269                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 270         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 271         final String interfaceName = key.getName();
 272         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 273         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 274             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 275             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 276             if (interfaceState != null) {
 277                 try {
 278                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 279                     final int ifIndex = interfaceState.getIfIndex();
 280                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 281                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 282                         // (the inventory tx goes in last)
 283                         List<ListenableFuture<?>> futures = new ArrayList<>();
 284                         //set of prefix used, as entry in prefix-to-interface datastore
 285                         // is prerequisite for refresh Fib to avoid race condition leading to
 286                         // missing remote next hop in bucket actions on bgp-vpn delete
 287                         Set<String> prefixListForRefreshFib = new HashSet<>();
 288                         ListenableFuture<?> confFuture =
 289                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 290                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 291                                     operTx -> futures.add(
 292                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 293                                             LOG.info(
 294                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 295                                                     + " on dpn {}",
 296                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 297                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 298                                                 confTx, operTx, invTx, interfaceState, vpnName,
 299                                                 prefixListForRefreshFib);
 300                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 301                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 302                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 303                                                 if (newAdjs != null) {
 304                                                     for (Adjacency adj : newAdjs) {
 305                                                         if (oldAdjs.contains(adj)) {
 306                                                             oldAdjs.remove(adj);
 307                                                         } else {
 308                                                             if (!isBgpVpnInternetVpn
 309                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 310                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 311                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 312                                                                     prefixListForRefreshFib);
 313                                                             }
 314                                                         }
 315                                                     }
 316                                                 }
 317                                                 for (Adjacency adj : oldAdjs) {
 318                                                     if (!isBgpVpnInternetVpn
 319                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 320                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 321                                                             operTx, confTx);
 322                                                     }
 323                                                 }
 324                                             }
 325                                         })))));
 326                         Futures.addCallback(confFuture,
 327                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 328                             MoreExecutors.directExecutor());
 329                         futures.add(confFuture);
 330                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 331                             MoreExecutors.directExecutor());
 332                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 333                             + " processed successfully", interfaceName, vpnName, dpnId);
 334                         return futures;
 335                     });
 336                 } catch (NumberFormatException | IllegalStateException e) {
 337                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 338                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 339                         vpnName, e);
 340                     return;
 341                 }
 342             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 343                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 344                     () -> {
 345                         ListenableFuture<?> future =
 346                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 347                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 348                                     confTx, vpnName);
 349                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 350                                     vpnName, vpnInterface.getDpnId());
 351                             });
 352                         LoggingFutures.addErrorLogging(future, LOG,
 353                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 354                         return Collections.singletonList(future);
 355                     });
 356             } else {
 357                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 358                     + " is not available", interfaceName, vpnName);
 359             }
 360         } else {
 361             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 362                     + " as vpn is pending delete", interfaceName, vpnName,
 363                 vpnInterface.getDpnId());
 364         }
 365     }
 366
 367     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 368     // see comments below.
 369     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 370     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 371             final int lportTag, boolean isInterfaceUp,
 372             TypedWriteTransaction<Configuration> writeConfigTxn,
 373             TypedWriteTransaction<Operational> writeOperTxn,
 374             TypedReadWriteTransaction<Configuration> writeInvTxn,
 375             Interface interfaceState, final String vpnName,
 376             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 377         final String interfaceName = vpnInterface.getName();
 378         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 379                 vpnName);
 380         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 381         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 382         if (!isInterfaceUp) {
 383             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 384                      interfaceName, dpId, vpnName);
 385             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 386             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 387                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 388                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 389                         dpId);
 390                 return;
 391             }
 392
 393             boolean waitForVpnInterfaceOpRemoval = false;
 394             if (opVpnInterface != null) {
 395                 String opVpnName = opVpnInterface.getVpnInstanceName();
 396                 String primaryInterfaceIp = null;
 397                 if (Objects.equals(opVpnName, vpnName)) {
 398                     // Please check if the primary VRF Entry does not exist for VPNInterface
 399                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 400                     // back to back
 401                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 402                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 403                     if (adjs == null) {
 404                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 405                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 406                                 vpnName);
 407                         return;
 408                     }
 409                     for (Adjacency adj : adjs) {
 410                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 411                             primaryInterfaceIp = adj.getIpAddress();
 412                             break;
 413                         }
 414                     }
 415                     if (primaryInterfaceIp == null) {
 416                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 417                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 418                                 dpId, vpnName);
 419                         return;
 420                     }
 421                     // Get the rd of the vpn instance
 422                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 423                     if (vrf != null) {
 424                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 425                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 426                         return;
 427                     }
 428                     waitForVpnInterfaceOpRemoval = true;
 429                 } else {
 430                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 431                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 432                 }
 433             }
 434             if (!waitForVpnInterfaceOpRemoval) {
 435                 // Add the VPNInterface and quit
 436                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 437                         null/*ipAddressSourceValuePair*/,
 438                         true /* add */);
 439                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 440                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 441                 if (!isBgpVpnInternetVpn) {
 442                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 443                 }
 444                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 445                         dpId, vpnName);
 446                 if (interfaceManager.isExternalInterface(interfaceName)) {
 447                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 448                         NwConstants.ADD_FLOW);
 449                 }
 450                 return;
 451             }
 452
 453             // FIB didn't get a chance yet to clean up this VPNInterface
 454             // Let us give it a chance here !
 455             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 456                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 457             try {
 458                 Runnable notifyTask = new VpnNotifyTask();
 459                 synchronized (notifyTask) {
 460                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 461                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 462                     // published here while holding the lock on it so this path will hit the wait before notify can be
 463                     // invoked.
 464                     vpnIntfMap.put(interfaceName, notifyTask);
 465                     try {
 466                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 467                     } catch (InterruptedException e) {
 468                         // Ignored
 469                     }
 470                 }
 471             } finally {
 472                 vpnIntfMap.remove(interfaceName);
 473             }
 474
 475             if (opVpnInterface != null) {
 476                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 477                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 478                         dpId, vpnName);
 479                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 480                 return;
 481             }
 482             // VPNInterface got removed, proceed with Add
 483             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 484                     interfaceName, dpId, vpnName);
 485             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 486                     null/*ipAddressSourceValuePair*/,
 487                     true /* add */);
 488             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 489                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 490             if (!isBgpVpnInternetVpn) {
 491                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 492             }
 493             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 494                     + " FIB to clean up", interfaceName, dpId, vpnName);
 495             if (interfaceManager.isExternalInterface(interfaceName)) {
 496                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 497                                lportTag, NwConstants.ADD_FLOW);
 498             }
 499         } else {
 500             try {
 501                 // Interface is retained in the DPN, but its Link Up.
 502                 // Advertise prefixes again for this interface to BGP
 503                 InstanceIdentifier<VpnInterface> identifier =
 504                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 505                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 506                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 507                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 508                 // Perform similar operation as interface add event for extraroutes.
 509                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 510                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 511                         LogicalDatastoreType.CONFIGURATION, path);
 512                 if (!optAdjacencies.isPresent()) {
 513                     LOG.trace("No config adjacencyKeyAdjacencyMap present for vpninterface {}", vpnInterface);
 514                     return;
 515                 }
 516                 Map<AdjacencyKey, Adjacency> adjacencyKeyAdjacencyMap = optAdjacencies.get().nonnullAdjacency();
 517                 for (Adjacency adjacency : adjacencyKeyAdjacencyMap.values()) {
 518                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 519                         continue;
 520                     }
 521                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 522                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 523                         continue;
 524                     }
 525                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 526                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 527                 }
 528             } catch (InterruptedException | ExecutionException e) {
 529                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 530                         interfaceName, vpnName, primaryRd, dpId);
 531             }
 532         }
 533     }
 534
 535     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 536         int lportTag, int addOrRemove) {
 537         Uuid extNetworkId;
 538         try {
 539             // vpn instance of ext-net interface is the network-id
 540             extNetworkId = new Uuid(vpnName);
 541         } catch (IllegalArgumentException e) {
 542             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 543                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 544             return;
 545         }
 546
 547         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 548         if (routerIds == null || routerIds.isEmpty()) {
 549             LOG.info("processExternalVpnInterface: No router is associated with {}."
 550                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 551                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 552             return;
 553         }
 554
 555         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 556                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 557         for (Uuid routerId : routerIds) {
 558             String routerName = routerId.getValue();
 559             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 560             if (Objects.equals(primarySwitch, dpId)) {
 561                 Routers router = vpnUtil.getExternalRouter(routerName);
 562                 if (router != null) {
 563                     if (addOrRemove == NwConstants.ADD_FLOW) {
 564                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 565                                 VpnUtil.getIpsListFromExternalIps(new ArrayList<>(router
 566                                                 .nonnullExternalIps().values())), router.getExtGwMacAddress(),
 567                                 dpId, interfaceName, lportTag);
 568                     } else {
 569                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 570                                 VpnUtil.getIpsListFromExternalIps(new ArrayList<>(router
 571                                         .nonnullExternalIps().values())),
 572                                 dpId, interfaceName, lportTag);
 573                     }
 574                 } else {
 575                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 576                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 577                             interfaceName, dpId, vpnName);
 578                 }
 579             }
 580         }
 581     }
 582
 583     // TODO Clean up the exception handling
 584     @SuppressWarnings("checkstyle:IllegalCatch")
 585     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 586                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 587                                                   String vpnName, String interfaceName) {
 588         if (rd == null) {
 589             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 590                   interfaceName, dpnId, vpnName);
 591             return;
 592         } else {
 593             if (rd.equals(vpnName)) {
 594                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 595                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 596                 return;
 597             }
 598         }
 599         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 600                 interfaceName, dpnId, vpnName, rd);
 601
 602         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 603         if (nextHopIp == null) {
 604             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 605                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 606                     rd, vpnName);
 607             return;
 608         }
 609
 610         try {
 611             //Read NextHops
 612             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 613             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 614                     LogicalDatastoreType.OPERATIONAL, path);
 615             if (adjacencies.isPresent()) {
 616                 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.get().getAdjacency();
 617                 if (nextHopsMap != null && !nextHopsMap.isEmpty()) {
 618                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 619                             + " rd {}", nextHopsMap, interfaceName, dpnId, vpnName, rd);
 620                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 621                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 622                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 623                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 624                     for (Adjacency nextHop : nextHopsMap.values()) {
 625                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 626                             continue;
 627                         }
 628                         String gatewayMac = null;
 629                         Uint32 label = Uint32.ZERO;
 630                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 631                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 632                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 633                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 634                                     .get();
 635                         } else {
 636                             label = nextHop.getLabel();
 637                         }
 638                         try {
 639                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 640                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 641                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 642                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 643                                     gatewayMac);
 644                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 645                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 646                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 647                         } catch (Exception e) {
 648                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 649                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 650                                     interfaceName, dpnId, e);
 651                         }
 652                     }
 653                 }
 654             }
 655         } catch (InterruptedException | ExecutionException e) {
 656             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 657                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 658         }
 659     }
 660
 661     // TODO Clean up the exception handling
 662     @SuppressWarnings("checkstyle:IllegalCatch")
 663     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 664                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 665                         TypedWriteTransaction<Operational> writeOperTx) {
 666         //Read NextHops
 667         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 668         String rd = vpnUtil.getVpnRd(interfaceName);
 669         if (rd == null) {
 670             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 671                 interfaceName, vpnName);
 672             return;
 673         } else {
 674             if (rd.equals(vpnName)) {
 675                 LOG.info(
 676                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 677                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 678                 return;
 679             }
 680         }
 681         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 682                vpnName, rd);
 683         Optional<AdjacenciesOp> adjacencies = Optional.empty();
 684         try {
 685             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 686                     path);
 687         } catch (InterruptedException | ExecutionException e) {
 688             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 689                     interfaceName, vpnName);
 690         }
 691         if (adjacencies.isPresent()) {
 692             Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.get().getAdjacency();
 693
 694             if (nextHopsMap != null && !nextHopsMap.isEmpty()) {
 695                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 696                         nextHopsMap, interfaceName, vpnName, rd);
 697                 for (Adjacency nextHop : nextHopsMap.values()) {
 698                     try {
 699                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 700                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 701                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 702                                     interfaceName, vpnName);
 703                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 704                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 705                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 706                                     interfaceName, vpnName);
 707                         } else if (nextHop.getNextHopIpList() != null) {
 708                             // Perform similar operation as interface delete event for extraroutes.
 709                             String allocatedRd = nextHop.getVrfId();
 710                             for (String nh : nextHop.getNextHopIpList()) {
 711                                 deleteExtraRouteFromCurrentAndImportingVpns(
 712                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 713                                     writeOperTx);
 714                             }
 715                         }
 716                     } catch (Exception e) {
 717                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 718                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 719                     }
 720                 }
 721             }
 722         }
 723     }
 724
 725     @SuppressWarnings("checkstyle:IllegalCatch")
 726     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 727                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 728                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 729                                                   TypedWriteTransaction<Operational> writeOperTxn,
 730                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 731                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 732             throws ExecutionException, InterruptedException {
 733         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 734         // Read NextHops
 735         Optional<VpnInterface> vpnInteface = Optional.empty();
 736         try {
 737             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 738             LogicalDatastoreType.CONFIGURATION, identifier);
 739         } catch (InterruptedException | ExecutionException e) {
 740             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 741                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 742         }
 743         Uuid intfnetworkUuid = null;
 744         NetworkType networkType = null;
 745         long segmentationId = -1L;
 746         Adjacencies adjacencies = null;
 747         if (vpnInteface.isPresent()) {
 748             intfnetworkUuid = vpnInteface.get().getNetworkId();
 749             networkType = vpnInteface.get().getNetworkType();
 750             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 751             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 752             if (adjacencies == null) {
 753                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 754                         null/*gwMac*/,  null/*gatewayIp*/, writeOperTxn);
 755                 return;
 756             }
 757         }
 758         // Get the rd of the vpn instance
 759         String nextHopIp = null;
 760         String gatewayIp = null;
 761         try {
 762             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 763         } catch (Exception e) {
 764             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 765                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 766         }
 767         List<String> nhList = new ArrayList<>();
 768         if (nextHopIp != null) {
 769             nhList.add(nextHopIp);
 770             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 771                     interfaceName, dpnId, vpnName, nhList);
 772         }
 773         Optional<String> gwMac = Optional.empty();
 774         String vpnInterfaceSubnetGwMacAddress = null;
 775         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 776         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 777         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 778         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 779         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 780         Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies != null ? adjacencies.getAdjacency()
 781                 : Collections.<AdjacencyKey, Adjacency>emptyMap();
 782         List<Adjacency> value = new ArrayList<>();
 783         for (Adjacency nextHop : nextHopsMap.values()) {
 784             String rd = primaryRd;
 785             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 786             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.InternetBGPVPN
 787                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 788                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 789                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 790                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 791                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 792                 continue;
 793             }
 794             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 795                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 796                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 797                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 798                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHopsMap {} "
 799                         + "on dpn {} for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 800
 801                 Prefixes prefixes = intfnetworkUuid != null
 802                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid, networkType,
 803                             segmentationId, prefixCue) :
 804                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 805                 writeOperTxn.mergeParentStructureMerge(VpnUtil.getPrefixToInterfaceIdentifier(
 806                         vpnUtil.getVpnId(vpnName), prefix), prefixes);
 807                 final Uuid subnetId = nextHop.getSubnetId();
 808
 809                 gatewayIp = nextHop.getSubnetGatewayIp();
 810                 if (gatewayIp == null) {
 811                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 812                     if (gatewayIpOptional.isPresent()) {
 813                         gatewayIp = gatewayIpOptional.get();
 814                     }
 815                 }
 816
 817                 if (gatewayIp != null) {
 818                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 819                     if (gwMac.isPresent()) {
 820                         // A valid mac-address is available for this subnet-gateway-ip
 821                         // Use this for programming ARP_RESPONDER table here.  And save this
 822                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 823                         // to populate L3_GW_MAC_TABLE there.
 824                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 825                                 gatewayIp, gwMac.get());
 826                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 827                     } else {
 828                         // A valid mac-address is not available for this subnet-gateway-ip
 829                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 830                         // Save this connected-mac-address as gateway-mac-address for the
 831                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 832                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 833                         if (gwMac.isPresent()) {
 834                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 835                                     NwConstants.ADD_FLOW, gwMac.get());
 836                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 837                                     gatewayIp, gwMac.get());
 838                         } else {
 839                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 840                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 841                                 + "gwIp {}",
 842                                 subnetId, interfaceName, vpnName, gatewayIp);
 843                         }
 844                     }
 845                 } else {
 846                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 847                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 848                         subnetId, interfaceName, vpnName);
 849                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 850                 }
 851                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHopsMap {} on dpn {}"
 852                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 853             } else {
 854                 //Extra route adjacency
 855                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 856                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 857                 // FIXME: separate this out somehow?
 858                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 859                 lock.lock();
 860                 try {
 861                     java.util.Optional<String> rdToAllocate = vpnUtil
 862                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 863                                     nextHop.getNextHopIpList().get(0), dpnId);
 864                     if (rdToAllocate.isPresent()) {
 865                         rd = rdToAllocate.get();
 866                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 867                             rd, prefix);
 868                     } else {
 869                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 870                         continue;
 871                     }
 872                 } finally {
 873                     lock.unlock();
 874                 }
 875                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 876                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 877                         interfaceName, dpnId);
 878             }
 879             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 880             // can be different from the gateway-mac-address within the VRFEntry as the
 881             // gateway-mac-address is a superset.
 882             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 883             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 884                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 885                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 886             Adjacency operationalAdjacency = null;
 887             try {
 888                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 889             } catch (NullPointerException e) {
 890                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 891                     input, e.getMessage());
 892                 return;
 893             }
 894             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 895                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 896                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 897                     writeConfigTxn);
 898             }
 899             value.add(operationalAdjacency);
 900         }
 901
 902         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 903         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 904                 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
 905
 906         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 907                 .setGatewayMac(gwMac.orElse(null)).setInterfaceName(interfaceName)
 908                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 909
 910         for (Adjacency nextHop : aug.nonnullAdjacency().values()) {
 911             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 912             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 913                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 914                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 915                 registeredPopulator.populateFib(input, writeConfigTxn);
 916             }
 917         }
 918     }
 919
 920     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 921                                               long lportTag, String gwMac, String gwIp,
 922                                               TypedWriteTransaction<Operational> writeOperTxn) {
 923         VpnInterfaceOpDataEntry opInterface =
 924               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
 925         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 926             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 927         writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
 928         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 929                 interfaceName, dpnId, vpnName);
 930     }
 931
 932     // TODO Clean up the exception handling
 933     @SuppressWarnings("checkstyle:IllegalCatch")
 934     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 935                                            StateTunnelList stateTunnelList,
 936                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 937                                            TypedWriteTransaction<Operational> writeOperTxn) {
 938
 939         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 940         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 941         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 942         Map<AdjacencyKey, Adjacency> keyAdjacencyMap =
 943             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency()
 944                     : Collections.<AdjacencyKey, Adjacency>emptyMap();
 945         if (keyAdjacencyMap.isEmpty()) {
 946             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 947                     vpnInterface, srcDpnId);
 948             return;
 949         }
 950         String prefix = null;
 951         List<Adjacency> value = new ArrayList<>();
 952         boolean isFibNextHopAddReqd = false;
 953         String vpnName = vpnInterface.getVpnInstanceName();
 954         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 955         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 956         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 957                 vpnInterface.getName(), vpnInterface.getDpnId(),
 958                 vpnInterface.getVpnInstanceName(), keyAdjacencyMap);
 959         for (Adjacency adj : keyAdjacencyMap.values()) {
 960             String rd = adj.getVrfId();
 961             rd = rd != null ? rd : vpnName;
 962             prefix = adj.getIpAddress();
 963             Uint32 label = adj.getLabel();
 964             List<String> nhList = Collections.singletonList(srcTepIp);
 965             List<String> nextHopList = adj.getNextHopIpList();
 966             // If TEP is added , update the nexthop of primary adjacency.
 967             // Secondary adj nexthop is already pointing to primary adj IP address.
 968             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 969                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 970                 if (nextHopList != null && !nextHopList.isEmpty()) {
 971                     /* everything right already */
 972                 } else {
 973                     isFibNextHopAddReqd = true;
 974                 }
 975             } else {
 976                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 977                 if (!vrfEntryOptional.isPresent()) {
 978                     continue;
 979                 }
 980                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 981                 if (!nhList.contains(srcTepIp)) {
 982                     nhList.add(srcTepIp);
 983                     isFibNextHopAddReqd = true;
 984                 }
 985                 value.add(adj);
 986             }
 987
 988             if (isFibNextHopAddReqd) {
 989                 updateLabelMapper(label, nhList);
 990                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 991                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 992                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 993                 // Update the VRF entry with nextHop
 994                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, true,
 995                         writeConfigTxn);
 996
 997                 //Get the list of VPN's importing this route(prefix) .
 998                 // Then update the VRF entry with nhList
 999                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1000                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1001                     String vpnRd = vpn.getVrfId();
1002                     if (vpnRd != null) {
1003                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, true,
1004                                 writeConfigTxn);
1005                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1006                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1007                             vpnInterface.getName(), srcDpnId, vpnName,
1008                             vpn.getVpnInstanceName(), vpnRd);
1009                     }
1010                 }
1011                 // Advertise the prefix to BGP only for external vpn
1012                 // since there is a nexthop change.
1013                 try {
1014                     if (!rd.equalsIgnoreCase(vpnName)) {
1015                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1016                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1017                                 null /*gatewayMacAddress*/);
1018                     }
1019                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1020                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1021                             srcDpnId, vpnName);
1022                 } catch (Exception ex) {
1023                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1024                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1025                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1026                 }
1027             }
1028         }
1029         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1030         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1031                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1032                 .addAugmentation(aug).build();
1033         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1034                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1035         writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
1036         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1037                 vpnInterface.getName(), srcDpnId, vpnName);
1038
1039     }
1040
1041     // TODO Clean up the exception handling
1042     @SuppressWarnings("checkstyle:IllegalCatch")
1043     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1044                                               StateTunnelList stateTunnelList,
1045                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1046                                               TypedWriteTransaction<Operational> writeOperTxn) {
1047
1048         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1049         List<Adjacency> adjList = adjacencies != null ? new ArrayList<>(adjacencies
1050                 .nonnullAdjacency().values()) : new ArrayList<>();
1051         String prefix = null;
1052         boolean isNextHopRemoveReqd = false;
1053         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1054         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1055         String vpnName = vpnInterface.getVpnInstanceName();
1056         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1057         String primaryRd = vpnUtil.getVpnRd(vpnName);
1058         if (adjList != null) {
1059             List<Adjacency> value = new ArrayList<>();
1060             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1061                     vpnInterface.getName(), vpnInterface.getDpnId(),
1062                     vpnInterface.getVpnInstanceName(), adjList);
1063             for (Adjacency adj : adjList) {
1064                 List<String> nhList = new ArrayList<>();
1065                 String rd = adj.getVrfId();
1066                 rd = rd != null ? rd : vpnName;
1067                 prefix = adj.getIpAddress();
1068                 List<String> nextHopList = adj.getNextHopIpList();
1069                 Uint32 label = adj.getLabel();
1070                 if (nextHopList != null && !nextHopList.isEmpty()) {
1071                     isNextHopRemoveReqd = true;
1072                 }
1073                 // If TEP is deleted , remove the nexthop from primary adjacency.
1074                 // Secondary adj nexthop will continue to point to primary adj IP address.
1075                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1076                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1077                 } else {
1078                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1079                     if (!vrfEntryOptional.isPresent()) {
1080                         continue;
1081                     }
1082                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1083                     if (nhList.contains(srcTepIp)) {
1084                         nhList.remove(srcTepIp);
1085                         isNextHopRemoveReqd = true;
1086                     }
1087                     value.add(adj);
1088                 }
1089
1090                 if (isNextHopRemoveReqd) {
1091                     updateLabelMapper(label, nhList);
1092                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1093                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1094                             prefix, nhList, vpnName,
1095                             vpnId, rd, vpnInterface.getName());
1096                     // Update the VRF entry with removed nextHop
1097                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, false,
1098                             writeConfigTxn);
1099
1100                     //Get the list of VPN's importing this route(prefix) .
1101                     // Then update the VRF entry with nhList
1102                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1103                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1104                         String vpnRd = vpn.getVrfId();
1105                         if (vpnRd != null) {
1106                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, false,
1107                                     writeConfigTxn);
1108                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1109                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1110                                     nhList, label, vpnInterface.getName(), srcDpnId,
1111                                     vpnName,
1112                                     vpn.getVpnInstanceName(), vpnRd);
1113                         }
1114                     }
1115
1116                     // Withdraw prefix from BGP only for external vpn.
1117                     try {
1118                         if (!rd.equalsIgnoreCase(vpnName)) {
1119                             bgpManager.withdrawPrefix(rd, prefix);
1120                         }
1121                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1122                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1123                                 vpnInterface.getName(), srcDpnId,
1124                                 vpnName);
1125                     } catch (Exception ex) {
1126                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1127                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1128                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1129                     }
1130                 }
1131             }
1132             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1133             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1134                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1135                     .addAugmentation(aug).build();
1136             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1137                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1138             writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
1139             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1140                          vpnInterface.getName(), srcDpnId, vpnName);
1141         }
1142     }
1143
1144     @SuppressWarnings("checkstyle:IllegalCatch")
1145     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1146         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1147         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry;
1148         String vpnRd = vpnUtil.getVpnRd(vpnName);
1149         try {
1150             VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1151             if (opDataEntry == null) {
1152                 LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}"
1153                         + " when check for vpns exporting the routes", vpnName, vpnRd);
1154                 return vpnsToExportRoute;
1155             }
1156             vpnInstanceOpDataEntry = opDataEntry;
1157         } catch (Exception re) {
1158             LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}"
1159                     + " to check for vpns exporting the routes", vpnName, vpnRd, re);
1160             return vpnsToExportRoute;
1161         }
1162         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1163             if (input.getVpnInstanceName() == null) {
1164                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1165                         input.getVrfId());
1166                 return false;
1167             }
1168             return !input.getVpnInstanceName().equals(vpnName);
1169         };
1170
1171         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1172             Iterable<String> commonRTs =
1173                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1174                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1175             return Iterators.size(commonRTs.iterator()) > 0;
1176         };
1177
1178         vpnsToExportRoute =
1179                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1180                         Collectors.toList());
1181         return vpnsToExportRoute;
1182     }
1183
1184     // TODO Clean up the exception handling
1185     @SuppressWarnings("checkstyle:IllegalCatch")
1186     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1187         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1188         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1189             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1190             if (vrfEntries != null) {
1191                 LoggingFutures.addErrorLogging(
1192                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1193                         for (VrfEntry vrfEntry : vrfEntries) {
1194                             try {
1195                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1196                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1197                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1198                                             + " is not a controller managed non intervpn link route. Ignoring.",
1199                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1200                                     continue;
1201                                 }
1202                                 String prefix = vrfEntry.getDestPrefix();
1203                                 String gwMac = vrfEntry.getGatewayMacAddress();
1204                                 vrfEntry.nonnullRoutePaths().values().forEach(routePath -> {
1205                                     String nh = routePath.getNexthopAddress();
1206                                     Uint32 label = routePath.getLabel();
1207                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1208                                         vrfEntry.getOrigin()))) {
1209                                         LOG.info(
1210                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1211                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1212                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1213                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1214                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1215                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1216                                             confTx);
1217                                     } else {
1218                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1219                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1220                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1221                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1222                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1223                                             confTx);
1224                                     }
1225                                 });
1226                             } catch (RuntimeException e) {
1227                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1228                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1229                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1230                             }
1231                         }
1232                     }), LOG, "Error handing VPN exporting routes");
1233             } else {
1234                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1235                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1236             }
1237         }
1238     }
1239
1240     @Override
1241     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1242         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1243         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1244         final String interfaceName = key.getName();
1245         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) {
1246             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1247             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1248         }
1249     }
1250
1251     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1252                                 final VpnInterface vpnInterface, final String vpnName,
1253                                 final String interfaceName) {
1254         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1255             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1256                 ListenableFuture<?> future =
1257                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1258                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1259                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1260                     });
1261                 LoggingFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1262                         vpnInterface.getName(), vpnName);
1263                 return Collections.singletonList(future);
1264             }, DJC_MAX_RETRIES);
1265         } else {
1266             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1267             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1268         }
1269     }
1270
1271     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1272     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1273                                            final VpnInterface vpnInterface, final String vpnName,
1274                                            final String interfaceName, final Interface interfaceState) {
1275         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1276                 vpnName, vpnInterface.getDpnId());
1277         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1278         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1279             () -> {
1280                 List<ListenableFuture<?>> futures = new ArrayList<>(3);
1281                 ListenableFuture<?> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1282                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1283                         writeOperTxn -> futures.add(
1284                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1285                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1286                                         interfaceName, vpnName);
1287                                 Uint64 dpId;
1288                                 int ifIndex;
1289                                 String gwMacAddress;
1290                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1291                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1292                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1293                                 try {
1294                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1295                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1296                                 } catch (InterruptedException | ExecutionException e) {
1297                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1298                                             interfaceName, vpnName);
1299                                     return;
1300                                 }
1301                                 if (interfaceState != null) {
1302                                     try {
1303                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1304                                     } catch (NumberFormatException | IllegalStateException e) {
1305                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1306                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1307                                                         + " from vpn interface op data store. ", interfaceName,
1308                                                 vpnInterface.getDpnId(), vpnName, e);
1309                                         dpId = Uint64.ZERO;
1310                                     }
1311                                     ifIndex = interfaceState.getIfIndex();
1312                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1313                                 } else {
1314                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1315                                             + " from vpn interface op.", interfaceName);
1316                                     if (optVpnInterface.isPresent()) {
1317                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1318                                         dpId = vpnOpInterface.getDpnId();
1319                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1320                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1321                                     } else {
1322                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1323                                                 + " as interfaceState and vpn interface op is not"
1324                                                 + " available", interfaceName, vpnName);
1325                                         return;
1326                                     }
1327                                 }
1328                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1329                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1330                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1331                                 LOG.info(
1332                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1333                                                 + "successfully",
1334                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1335                             })))));
1336                 futures.add(configFuture);
1337                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1338                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1339                 return futures;
1340             }, DJC_MAX_RETRIES);
1341     }
1342
1343     protected void processVpnInterfaceDown(Uint64 dpId,
1344                                            String interfaceName,
1345                                            int lportTag,
1346                                            String gwMac,
1347                                            VpnInterfaceOpDataEntry vpnOpInterface,
1348                                            boolean isInterfaceStateDown,
1349                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1350                                            TypedWriteTransaction<Operational> writeOperTxn,
1351                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1352             throws ExecutionException, InterruptedException {
1353         if (vpnOpInterface == null) {
1354             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1355                     + " as it is not available in operational data store", interfaceName, dpId);
1356             return;
1357         }
1358         final String vpnName = vpnOpInterface.getVpnInstanceName();
1359         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1360                                                     interfaceName, vpnName);
1361         if (!isInterfaceStateDown) {
1362             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1363             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1364             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1365             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1366                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1367             if (interfaceManager.isExternalInterface(interfaceName)) {
1368                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1369                     NwConstants.DEL_FLOW);
1370             }
1371             if (!isBgpVpnInternetVpn) {
1372                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1373             }
1374             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1375                     + " successful", interfaceName, dpId, vpnName);
1376         } else {
1377             // Interface is retained in the DPN, but its Link Down.
1378             // Only withdraw the prefixes for this interface from BGP
1379             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1380         }
1381     }
1382
1383     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1384                                           final String vpnName, final Uint32 vpnId, String gwMac,
1385                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1386                                           TypedWriteTransaction<Operational> writeOperTxn,
1387                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1388             throws ExecutionException, InterruptedException {
1389         //Read NextHops
1390         try {
1391             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1392                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1393             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1394                     SingleTransactionDataBroker.syncReadOptional(dataBroker,
1395                             LogicalDatastoreType.OPERATIONAL, identifier);
1396             boolean isNonPrimaryAdjIp = false;
1397             String primaryRd = vpnUtil.getVpnRd(vpnName);
1398             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1399                     interfaceName, dpnId, vpnName, primaryRd);
1400             if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1401                 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1402                         + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1403                 return;
1404             }
1405             AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1406
1407             if (adjacencies != null && adjacencies.getAdjacency() != null) {
1408                 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.nonnullAdjacency();
1409                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1410                         interfaceName, dpnId, vpnName, nextHopsMap);
1411                 for (Adjacency nextHop : nextHopsMap.values()) {
1412                     if (nextHop.isPhysNetworkFunc()) {
1413                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1414                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1415                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1416                                 null, null/*writeCfgTxn*/);
1417                     } else {
1418                         String rd = nextHop.getVrfId();
1419                         List<String> nhList;
1420                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1421                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1422                             isNonPrimaryAdjIp = Boolean.TRUE;
1423                         } else {
1424                             // This is a primary adjacency
1425                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1426                                     : emptyList();
1427                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1428                                     vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1429                                     interfaceName, writeInvTxn);
1430                             isNonPrimaryAdjIp = Boolean.FALSE;
1431                         }
1432                         if (!nhList.isEmpty()) {
1433                             if (Objects.equals(primaryRd, vpnName)) {
1434                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1435                                 //remove from FIB directly
1436                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1437                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1438                             } else {
1439                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1440                                         interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn);
1441                             }
1442                         } else {
1443                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1444                                             + " interface {}", nextHop.getIpAddress(), rd,
1445                                     nextHop.getAdjacencyType().toString(), interfaceName);
1446                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1447                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1448                         }
1449                     }
1450                     String ip = nextHop.getIpAddress().split("/")[0];
1451                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1452                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1453                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1454                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1455                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1456                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1457                     }
1458                     // Remove the MIP-IP from VpnPortIpToPort.
1459                     if (isNonPrimaryAdjIp) {
1460                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1461                         if (persistedIp != null && persistedIp.isLearntIp()
1462                                 && persistedIp.getPortName().equals(interfaceName)) {
1463                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1464                             LOG.info(
1465                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1466                                             + "from VpnPortipToPort",
1467                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1468                         }
1469                     }
1470                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1471                     if (vpnPortipToPort != null) {
1472                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1473                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1474                                  + "Interface {} ip {} on dpn {} for vpn {}",
1475                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1476                     }
1477                 }
1478             } else {
1479                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1480                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1481                         + " Removing it.", interfaceName, vpnName, dpnId);
1482                 writeOperTxn.delete(identifier);
1483             }
1484         } catch (InterruptedException | ExecutionException e) {
1485             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1486                     interfaceName, dpnId, vpnName);
1487         }
1488     }
1489
1490     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1491                                                             String interfaceName, Uint64 dpnId,
1492                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1493                                                             TypedWriteTransaction<Operational> writeOperTx) {
1494         return nh -> {
1495             String primaryRd = vpnUtil.getVpnRd(vpnName);
1496             String prefix = nextHop.getIpAddress();
1497             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1498             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1499                     nextHop, vpnName, interfaceName, dpnId);
1500             // FIXME: separate this out somehow?
1501             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1502             lock.lock();
1503             try {
1504                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1505                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1506                     //If extra-route is present behind at least one VM, then do not remove or update
1507                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1508                     //prefixtointerface DS
1509                     return;
1510                 }
1511                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1512                         writeConfigTxn);
1513             } finally {
1514                 lock.unlock();
1515             }
1516             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1517                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1518                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1519         };
1520     }
1521
1522     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1523                                            Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp,
1524                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1525                                            TypedWriteTransaction<Operational> writeOperTx) {
1526         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1527                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1528         nhList.forEach(nh -> {
1529             //IRT: remove routes from other vpns importing it
1530             if (isNonPrimaryAdjIp) {
1531                 removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1532             } else {
1533                 vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1534                         nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1535             }
1536             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1537                 String vpnRd = vpn.getVrfId();
1538                 if (vpnRd != null) {
1539                     fibManager.removeOrUpdateFibEntry(vpnRd,
1540                             nextHop.getIpAddress(), nh, writeConfigTxn);
1541                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1542                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1543                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1544                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1545                 }
1546             }
1547         });
1548     }
1549
1550     @SuppressWarnings("checkstyle:IllegalCatch")
1551     private void removeLearntPrefixFromBGP(String rd, String prefix, String nextHop,
1552                                            TypedWriteTransaction<Configuration> writeConfigTxn) {
1553         try {
1554             if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) {
1555                 LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this"
1556                         + " operation", prefix, nextHop, rd);
1557                 return;
1558             }
1559             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}",
1560                     rd, prefix, nextHop);
1561             fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn);
1562             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1563             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}",
1564                     rd, prefix, nextHop);
1565         } catch (Exception e) {
1566             LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e);
1567         }
1568     }
1569
1570     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1571                                                  int lportTag, String gwMac, String gwIp, String interfaceName,
1572                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1573             throws ExecutionException, InterruptedException {
1574         final Uuid subnetId = nextHop.getSubnetId();
1575         if (nextHop.getSubnetGatewayMacAddress() == null) {
1576             // A valid mac-address was not available for this subnet-gateway-ip
1577             // So a connected-mac-address was used for this subnet and we need
1578             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1579             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1580         }
1581         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1582                 subnetId);
1583     }
1584
1585     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1586                                                   String interfaceName) {
1587         // This is either an extra-route (or) a learned IP via subnet-route
1588         List<String> nhList = null;
1589         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1590         if (nextHopIp == null || nextHopIp.isEmpty()) {
1591             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1592                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1593                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1594                     vpnName);
1595             nhList = emptyList();
1596         } else {
1597             nhList = Collections.singletonList(nextHopIp);
1598         }
1599         return nhList;
1600     }
1601
1602     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1603         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1604         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1605         // else use connected interface
1606         if (gwPort != null && gwPort.isSubnetIp()) {
1607             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1608                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1609             return Optional.of(gwPort.getMacAddress());
1610         }
1611         return Optional.empty();
1612     }
1613
1614     @Override
1615     public void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1616         final VpnInterface update) {
1617         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1618         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1619                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1620         if (original.equals(update)) {
1621             LOG.info("update: original {} update {} are same. No update required.", original, update);
1622             return;
1623         }
1624         final String vpnInterfaceName = update.getName();
1625         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1626         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1627         //handles switching between <internal VPN - external VPN>
1628         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1629             List<ListenableFuture<?>> futures = new ArrayList<>();
1630             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1631                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1632                                 + "to newVpn(s) {}",
1633                         original.getName(), dpnId,
1634                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(
1635                                 new ArrayList<>(original.nonnullVpnInstanceNames().values())),
1636                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(
1637                                 new ArrayList<>(update.nonnullVpnInstanceNames().values())));
1638                 return emptyList();
1639             }
1640             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1641             return futures;
1642         });
1643     }
1644
1645     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1646                                                            VpnInterface original, VpnInterface update,
1647                                                            List<ListenableFuture<?>> futures) {
1648         boolean isVpnInstanceUpdate = false;
1649         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1650         final String interfaceName = key.getName();
1651         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1652         List<String> oldVpnListCopy = new ArrayList<>();
1653         oldVpnListCopy.addAll(oldVpnList);
1654         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1655         List<String> newVpnListCopy = new ArrayList<>();
1656         newVpnListCopy.addAll(newVpnList);
1657
1658         oldVpnList.removeAll(newVpnList);
1659         newVpnList.removeAll(oldVpnListCopy);
1660         //This block will execute only on if there is a change in the VPN Instance.
1661         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1662             /*
1663              * Internet BGP-VPN Instance update with single router:
1664              * ====================================================
1665              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1666              *     1st VPN Instance : router VPN or external BGP-VPN.
1667              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1668              *
1669              * VPN Instance UPDATE:
1670              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1671              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1672              *
1673              * External BGP-VPN Instance update with single router:
1674              * ====================================================
1675              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1676              *
1677              * Updated VPN Instance will be always either internal router VPN to
1678              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1679              *
1680              * VPN Instance UPDATE:
1681              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1682              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1683              *
1684              * Dual Router VPN Instance Update:
1685              * ================================
1686              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1687              *
1688              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1689              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1690              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1691              *
1692              * Dual Router --> Associated with common external BGP-VPN Instance.
1693              * 1st router and 2nd router are getting associated with single External BGP-VPN
1694              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1695              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1696              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1697              *
1698              *
1699              */
1700             isVpnInstanceUpdate = true;
1701             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1702                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1703                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1704                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1705                     List<String> externalBgpVpnList = new ArrayList<>();
1706                     for (String newVpnName : newVpnListCopy) {
1707                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1708                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1709                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1710                                 .BgpvpnType.BGPVPN) {
1711                             externalBgpVpnList.add(newVpnName);
1712                             break;
1713                         }
1714                     }
1715                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1716                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1717                             externalBgpVpnList, oldVpnListCopy, futures);
1718
1719                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1720                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1721                     //Identify the router VPN Instance and pass that value as oldVpnList
1722                     List<String> routerVpnList = new ArrayList<>();
1723                     for (String newVpnName : newVpnListCopy) {
1724                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1725                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1726                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1727                                 .BgpvpnType.BGPVPN) {
1728                             routerVpnList.add(newVpnName);
1729                             break;
1730                         }
1731                     }
1732                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1733                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1734                             newVpnList, oldVpnListCopy, futures);
1735
1736                 } else {
1737                     //Handle remaining use cases.
1738                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1739                             oldVpnListCopy, futures);
1740                 }
1741             } else {
1742                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1743                         oldVpnListCopy, futures);
1744             }
1745         }
1746         return isVpnInstanceUpdate;
1747     }
1748
1749     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1750                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1751                                          List<String> newVpnList, List<String> oldVpnListCopy,
1752                                          List<ListenableFuture<?>> futures) {
1753         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1754         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1755                 ? new ArrayList<>(origAdjs.getAdjacency().values()) : new ArrayList<>();
1756         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1757         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1758                 ? new ArrayList<>(updateAdjs.getAdjacency().values()) : new ArrayList<>();
1759
1760         boolean isOldVpnRemoveCallExecuted = false;
1761         for (String oldVpnName : oldVpnList) {
1762             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1763                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1764             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1765             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1766                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1767                     newVpnList);
1768             isOldVpnRemoveCallExecuted = true;
1769         }
1770         //Wait for previous interface bindings to be removed
1771         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1772             try {
1773                 Thread.sleep(2000);
1774             } catch (InterruptedException e) {
1775                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1776             }
1777         }
1778         for (String newVpnName : newVpnList) {
1779             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1780             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1781                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1782                         + "onto vpnName {} ", interfaceName, newVpnName);
1783                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1784                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1785                                 + "from oldVpn(s) {} to newVpn {} ",
1786                         interfaceName, oldVpnListCopy, newVpnName);
1787                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1788                  * Use Case:
1789                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1790                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1791                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1792                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1793                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1794                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1795                  *     router VPN as well as Internet BGP-VPN.
1796                  *
1797                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1798                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1799                  *                                       update in existing router VPN instance.
1800                  */
1801                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1802                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1803                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1804                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1805                 }
1806             } else {
1807                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1808                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1809                         interfaceName, oldVpnListCopy, newVpnName);
1810             }
1811         }
1812     }
1813
1814     // TODO Clean up the exception handling
1815     @SuppressWarnings("checkstyle:IllegalCatch")
1816     private List<ListenableFuture<?>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1817                                                                     String vpnInterfaceName,
1818                                                                     List<ListenableFuture<?>> futures) {
1819         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1820         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1821                 != null ? new ArrayList<>(origAdjs.getAdjacency().values()) : new ArrayList<>();
1822         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1823         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1824                 != null ? new ArrayList<>(updateAdjs.getAdjacency().values()) : new ArrayList<>();
1825
1826         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1827         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames().values()) {
1828             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1829             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1830             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1831             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1832             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1833                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1834                 //set of prefix used as entry in prefix-to-interface datastore
1835                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1836                 // in bucket actions on bgp-vpn delete
1837                 Set<String> prefixListForRefreshFib = new HashSet<>();
1838                 ListenableFuture<?> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1839                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1840                         operTx -> {
1841                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1842                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1843                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1844                                     update.getName(), newVpnName);
1845                             //handle both addition and removal of adjacencies
1846                             // currently, new adjacency may be an extra route
1847                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1848                             if (!oldAdjs.equals(newAdjs)) {
1849                                 for (Adjacency adj : copyNewAdjs) {
1850                                     if (copyOldAdjs.contains(adj)) {
1851                                         copyOldAdjs.remove(adj);
1852                                     } else {
1853                                         // add new adjacency
1854                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1855                                             try {
1856                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1857                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1858                                             } catch (RuntimeException e) {
1859                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1860                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1861                                             }
1862                                         }
1863                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1864                                             + " added to vpn interface {} on vpn {} dpnId {}",
1865                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1866                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1867                                     }
1868                                 }
1869                                 for (Adjacency adj : copyOldAdjs) {
1870                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1871                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1872                                             && !adj.isPhysNetworkFunc()) {
1873                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1874                                                 confTx);
1875                                             //remove FIB entry
1876                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1877                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1878                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1879                                             //remove BGP entry
1880                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1881                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1882                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1883                                             }
1884                                         } else {
1885                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1886                                                 operTx, confTx);
1887                                         }
1888                                     }
1889                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1890                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1891                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1892                                 }
1893                             }
1894                         })));
1895                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1896                     MoreExecutors.directExecutor());
1897                 futures.add(configTxFuture);
1898                 for (ListenableFuture<?> future : futures) {
1899                     LoggingFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1900                             update.getName(), update.getVpnInstanceNames());
1901                 }
1902             } else {
1903                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1904                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1905             }
1906         }
1907         return futures;
1908     }
1909
1910     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1911         final String labelStr = requireNonNull(label, "updateLabelMapper: label cannot be null or empty!").toString();
1912         // FIXME: separate this out somehow?
1913         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1914         lock.lock();
1915         try {
1916             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1917                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1918             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1919                 LogicalDatastoreType.OPERATIONAL, lriIid);
1920             if (opResult.isPresent()) {
1921                 LabelRouteInfo labelRouteInfo =
1922                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1923                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1924                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1925             }
1926             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1927                     nextHopIpList);
1928         } catch (InterruptedException | ExecutionException e) {
1929             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1930                     nextHopIpList);
1931         } catch (TransactionCommitFailedException e) {
1932             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1933                     nextHopIpList);
1934         } finally {
1935             lock.unlock();
1936         }
1937     }
1938
1939     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1940         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1941
1942         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1943         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1944                 .addAugmentation(route).build();
1945         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1946         InstanceIdentifierBuilder<VrfTables> idBuilder =
1947             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1948         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1949         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1950         if (writeConfigTxn != null) {
1951             writeConfigTxn.mergeParentStructureMerge(vrfTableId, vrfTableNew);
1952         } else {
1953             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1954         }
1955         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1956                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1957     }
1958
1959     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1960                                            Adjacency adj, Uint64 dpnId,
1961                                            TypedWriteTransaction<Operational> writeOperTxn,
1962                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1963                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1964                                            Set<String> prefixListForRefreshFib)
1965             throws ExecutionException, InterruptedException {
1966         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1967         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1968         try {
1969             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1970                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1971             if (optVpnInterface.isPresent()) {
1972                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1973                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1974                 String vpnName = currVpnIntf.getVpnInstanceName();
1975                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1976                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1977                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1978                         LogicalDatastoreType.OPERATIONAL, adjPath);
1979                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1980                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1981                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1982                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1983                 List<Adjacency> adjacencies = new ArrayList<>();
1984                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1985                     adjacencies.addAll(optAdjacencies.get().getAdjacency().values());
1986                 }
1987                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1988                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1989                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1990                 Adjacency operationalAdjacency = null;
1991                 //Handling dual stack neutron port primary adjacency
1992                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1993                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1994                             currVpnIntf.getName(), vpnName);
1995                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1996                             currVpnIntf.getName());
1997                     if (interfaceState != null) {
1998                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1999                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
2000                             prefixListForRefreshFib);
2001                     }
2002                 }
2003                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
2004                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
2005                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
2006                             : RouteOrigin.STATIC;
2007                     String nh = adj.getNextHopIpList().get(0);
2008                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
2009                     // FIXME: separate out to somehow?
2010                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
2011                     lock.lock();
2012                     try {
2013                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
2014                                 vpnId, null, prefix, vpnName, nh, dpnId);
2015                         if (rdToAllocate.isPresent()) {
2016                             input.setRd(rdToAllocate.get());
2017                             operationalAdjacency = populator.createOperationalAdjacency(input);
2018                             int label = operationalAdjacency.getLabel().intValue();
2019                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
2020                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
2021                                     currVpnIntf.getName(), operationalAdjacency, encapType,
2022                                     prefixListForRefreshFib, writeConfigTxn);
2023                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
2024                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
2025                                     vpnName, label, currVpnIntf.getName(), dpnId);
2026                         } else {
2027                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2028                                     vpnName, prefix);
2029                             return;
2030                         }
2031                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2032                         // Keeping the MPLS check for now.
2033                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2034                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2035                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2036                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
2037                             vpnsToImportRoute.forEach(vpn -> {
2038                                 if (vpn.getVrfId() != null) {
2039                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2040                                             vpnId, prefix,
2041                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2042                                             .ifPresent(
2043                                                 rds -> vpnManager.addExtraRoute(
2044                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2045                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2046                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2047                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2048                                 }
2049                             });
2050                         }
2051                     } finally {
2052                         lock.unlock();
2053                     }
2054                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2055                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2056                             currVpnIntf.getName(), vpnName);
2057
2058                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2059                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2060                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2061                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2062                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2063                             Prefixes.PrefixCue.PhysNetFunc);
2064                     if (vpnIntefaceConfig.isPresent()) {
2065                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2066                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2067                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2068                     }
2069
2070                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2071
2072                     writeOperTxn.mergeParentStructureMerge(
2073                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2074                                     prefix), pnfPrefix);
2075
2076                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2077                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2078                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2079                             RouteOrigin.LOCAL, writeConfigTxn);
2080
2081                     input.setRd(adj.getVrfId());
2082                 }
2083                 if (operationalAdjacency == null) {
2084                     operationalAdjacency = populator.createOperationalAdjacency(input);
2085                 }
2086                 adjacencies.add(operationalAdjacency);
2087                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2088                 VpnInterfaceOpDataEntry newVpnIntf =
2089                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2090                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2091                                 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2092                 writeOperTxn.mergeParentStructureMerge(identifier, newVpnIntf);
2093             }
2094         } catch (InterruptedException | ExecutionException e) {
2095             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2096                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2097         }
2098     }
2099
2100     @Nullable
2101     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2102         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2103         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2104     }
2105
2106     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2107                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2108                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2109         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2110         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2111         try {
2112             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2113                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2114             if (optVpnInterface.isPresent()) {
2115                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2116                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2117                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2118                         LogicalDatastoreType.OPERATIONAL, path);
2119                 if (optAdjacencies.isPresent()) {
2120                     Map<AdjacencyKey, Adjacency> keyAdjacencyMap = optAdjacencies.get().getAdjacency();
2121
2122                     if (keyAdjacencyMap != null && !keyAdjacencyMap.isEmpty()) {
2123                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", keyAdjacencyMap);
2124                         for (Adjacency adjacency : keyAdjacencyMap.values()) {
2125                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2126                                 String rd = adjacency.getVrfId();
2127                                 if (adj.getNextHopIpList() != null) {
2128                                     for (String nh : adj.getNextHopIpList()) {
2129                                         deleteExtraRouteFromCurrentAndImportingVpns(
2130                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2131                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2132                                     }
2133                                 } else if (adj.isPhysNetworkFunc()) {
2134                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2135                                             adj.getIpAddress(), adj.getSubnetId());
2136                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2137                                             null, writeConfigTxn);
2138                                 }
2139                                 break;
2140                             }
2141
2142                         }
2143                     }
2144                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2145                             dpnId, adj.getVrfId());
2146                 } else {
2147                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2148                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2149                 }
2150             }
2151         } catch (InterruptedException | ExecutionException e) {
2152             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2153                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2154         }
2155     }
2156
2157     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2158                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2159                                     TypedWriteTransaction<Operational> writeOperTx) {
2160         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2161                 destination, nextHop, vpnName, intfName, rd);
2162         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2163         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2164         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2165             String vpnRd = vpn.getVrfId();
2166             if (vpnRd != null) {
2167                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2168                         destination, nextHop, vpnName, intfName, vpnRd);
2169                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2170                         writeOperTx);
2171             }
2172         }
2173     }
2174
2175     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2176         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2177             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2178             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2179     }
2180
2181     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2182         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2183             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2184     }
2185
2186     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2187                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2188         if (vpnInterface == null) {
2189             return;
2190         }
2191         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2192         if (adjs == null) {
2193             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2194                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2195             return;
2196         }
2197         for (Adjacency adj : adjs) {
2198             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2199                 String primaryInterfaceIp = adj.getIpAddress();
2200                 String macAddress = adj.getMacAddress();
2201                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2202
2203                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2204                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2205                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
2206                     LOG.error(
2207                             "createFibEntryForRouterInterface: Unable to retrieve label for vpn pool {}, "
2208                                     + "vpninterface {}, vpn {}, rd {}",
2209                             VpnConstants.VPN_IDPOOL_NAME, interfaceName, vpnName, primaryRd);
2210                     return;
2211                 }
2212                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2213                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2214                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2215                         routerInt, label, writeConfigTxn);
2216                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2217                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2218                         macAddress);
2219             } else {
2220                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2221                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2222                         interfaceName, primaryRd, vpnName);
2223             }
2224         }
2225     }
2226
2227     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2228             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2229         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2230         String rd = vpnUtil.getVpnRd(vpnName);
2231         if (adjs != null) {
2232             Map<AdjacencyKey, Adjacency> keyAdjacencyMap = adjs.nonnullAdjacency();
2233             for (Adjacency adj : keyAdjacencyMap.values()) {
2234                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2235                     String primaryInterfaceIp = adj.getIpAddress();
2236                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2237                     fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2238                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2239                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2240                 }
2241             }
2242         } else {
2243             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2244                     vpnInterface.getName(), rd);
2245         }
2246     }
2247
2248     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2249         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2250         final String interfaceName = key.getName();
2251         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2252                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2253         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2254                   intefaceData.identifier, vpnName);
2255     }
2256
2257     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2258                                               VpnInterface vpnInterface, String vpnName) {
2259         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2260                .get(vpnName);
2261         if (vpnInterfaces == null) {
2262             vpnInterfaces = new ConcurrentLinkedQueue<>();
2263         }
2264         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2265         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2266         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2267                 vpnInterface.getName(), vpnName);
2268     }
2269
2270     public boolean isVpnInstanceReady(String vpnInstanceName) {
2271         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2272         if (vpnRd == null) {
2273             return false;
2274         }
2275         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2276
2277         return vpnInstanceOpDataEntry != null;
2278     }
2279
2280     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2281         // FIXME: separate out to somehow?
2282         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2283         lock.lock();
2284         try {
2285             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2286                     unprocessedVpnInterfaces.get(vpnInstanceName);
2287             if (vpnInterfaces != null) {
2288                 while (!vpnInterfaces.isEmpty()) {
2289                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2290                     if (hasVpnInstanceCreatedSuccessfully) {
2291                         processSavedInterface(savedInterface, vpnInstanceName);
2292                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2293                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2294                     } else {
2295                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2296                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2297                     }
2298                 }
2299             } else {
2300                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2301             }
2302         } finally {
2303             lock.unlock();
2304         }
2305     }
2306
2307     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2308             VpnInterface vpnInterface) {
2309         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2310         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2311         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2312         lock.lock();
2313         try {
2314             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2315                     unprocessedVpnInterfaces.get(firstVpnName);
2316             if (vpnInterfaces != null) {
2317                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2318                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2319                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2320                 }
2321             } else {
2322                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2323             }
2324         } finally {
2325             lock.unlock();
2326         }
2327     }
2328
2329     public void vpnInstanceIsReady(String vpnInstanceName) {
2330         processSavedInterfaces(vpnInstanceName, true);
2331     }
2332
2333     public void vpnInstanceFailed(String vpnInstanceName) {
2334         processSavedInterfaces(vpnInstanceName, false);
2335     }
2336
2337     private static class UnprocessedVpnInterfaceData {
2338         InstanceIdentifier<VpnInterface> identifier;
2339         VpnInterface vpnInterface;
2340
2341         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2342             this.identifier = identifier;
2343             this.vpnInterface = vpnInterface;
2344         }
2345
2346         @Override
2347         public int hashCode() {
2348             final int prime = 31;
2349             int result = 1;
2350             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2351             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2352             return result;
2353         }
2354
2355         @Override
2356         public boolean equals(Object obj) {
2357             if (this == obj) {
2358                 return true;
2359             }
2360             if (obj == null) {
2361                 return false;
2362             }
2363             if (getClass() != obj.getClass()) {
2364                 return false;
2365             }
2366             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2367             if (!Objects.equals(identifier, other.identifier)) {
2368                 return false;
2369             }
2370             if (!Objects.equals(vpnInterface, other.vpnInterface)) {
2371                 return false;
2372             }
2373             return true;
2374         }
2375     }
2376
2377     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2378         String primaryRd = vpnUtil.getVpnRd(vpnName);
2379         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2380         if (vpnInstanceOpData == null || vpnInstanceOpData.getVpnToDpnList() == null) {
2381             return;
2382         }
2383         List<VpnToDpnList> vpnToDpnLists = new ArrayList<>(vpnInstanceOpData.getVpnToDpnList().values());
2384         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2385             return;
2386         }
2387         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2388         vpnToDpnLists.forEach(vpnToDpnList -> {
2389             if (vpnToDpnList.getVpnInterfaces() == null) {
2390                 return;
2391             }
2392             vpnToDpnList.nonnullVpnInterfaces().values().forEach(vpnInterface -> {
2393                 try {
2394                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2395                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2396                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2397                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2398                     if (!vpnInterfaceOptional.isPresent()) {
2399                         return;
2400                     }
2401                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2402                             vpnInterface.getInterfaceName());
2403                     if (configVpnAdjacencies == null) {
2404                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2405                         return;
2406                     }
2407                     List<Adjacency> operationVpnAdjacencies = new ArrayList<>(vpnInterfaceOptional.get()
2408                             .augmentation(AdjacenciesOp.class).nonnullAdjacency().values());
2409                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2410                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2411                     // in operational DS. These unprocessed adjacencies will be handled below.
2412                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2413                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2414                     configVpnAdjacencies.stream()
2415                         .filter(adjacency -> operationVpnAdjacencies.stream()
2416                                 .noneMatch(operationalAdjacency ->
2417                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2418                         .forEach(adjacency -> {
2419                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2420                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2421                                 () -> {
2422                                     // TODO Deal with sequencing — the config tx must only submitted
2423                                     // if the oper tx goes in
2424                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2425                                         List<ListenableFuture<?>> futures = new ArrayList<>();
2426                                         futures.add(
2427                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2428                                                 //set of prefix used, as entry in prefix-to-interface datastore
2429                                                 // is prerequisite for refresh Fib to avoid race condition leading
2430                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2431                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2432                                                 ListenableFuture<?> configTxFuture =
2433                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2434                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2435                                                             primaryRd, adjacency,
2436                                                             vpnInterfaceOptional.get().getDpnId(),
2437                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2438                                                 Futures.addCallback(configTxFuture,
2439                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2440                                                     MoreExecutors.directExecutor());
2441                                                 futures.add(configTxFuture);
2442                                             }));
2443                                         return futures;
2444                                     } else {
2445                                         return emptyList();
2446                                     }
2447                                 });
2448                         });
2449                 } catch (InterruptedException | ExecutionException e) {
2450                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2451                             vpnName, primaryRd);
2452                 }
2453             });
2454         });
2455     }
2456
2457     private class PostVpnInterfaceWorker implements FutureCallback<Object> {
2458         private final String interfaceName;
2459         private final boolean add;
2460         private final String txnDestination;
2461
2462         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2463             this.interfaceName = interfaceName;
2464             this.add = add;
2465             this.txnDestination = transactionDest;
2466         }
2467
2468         @Override
2469         public void onSuccess(Object voidObj) {
2470             if (add) {
2471                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2472                         interfaceName, txnDestination);
2473             } else {
2474                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2475             }
2476         }
2477
2478         @Override
2479         public void onFailure(Throwable throwable) {
2480             if (add) {
2481                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2482                         interfaceName, txnDestination, throwable);
2483             } else {
2484                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2485                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2486             }
2487         }
2488     }
2489
2490     private class VpnInterfaceCallBackHandler implements FutureCallback<Object> {
2491         private final String primaryRd;
2492         private final Set<String> prefixListForRefreshFib;
2493
2494         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2495             this.primaryRd = primaryRd;
2496             this.prefixListForRefreshFib = prefixListForRefreshFib;
2497         }
2498
2499         @Override
2500         public void onSuccess(Object voidObj) {
2501             prefixListForRefreshFib.forEach(prefix -> {
2502                 fibManager.refreshVrfEntry(primaryRd, prefix);
2503             });
2504         }
2505
2506         @Override
2507         public void onFailure(Throwable throwable) {
2508             LOG.debug("write Tx config operation failed", throwable);
2509         }
2510     }
2511 }