vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Iterators;
  16 import com.google.common.util.concurrent.FutureCallback;
  17 import com.google.common.util.concurrent.Futures;
  18 import com.google.common.util.concurrent.ListenableFuture;
  19 import com.google.common.util.concurrent.MoreExecutors;
  20 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Optional;
  28 import java.util.Set;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.locks.ReentrantLock;
  33 import java.util.function.Consumer;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.eclipse.jdt.annotation.Nullable;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.Datastore.Configuration;
  42 import org.opendaylight.genius.infra.Datastore.Operational;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  45 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  46 import org.opendaylight.genius.infra.TypedWriteTransaction;
  47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  48 import org.opendaylight.genius.mdsalutil.NWUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  51 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  52 import org.opendaylight.genius.utils.JvmGlobalLocks;
  53 import org.opendaylight.infrautils.caches.CacheProvider;
  54 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  55 import org.opendaylight.infrautils.utils.concurrent.Executors;
  56 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  57 import org.opendaylight.mdsal.binding.api.DataBroker;
  58 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  59 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  60 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  61 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  62 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  63 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  64 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  65 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  66 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  67 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  68 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  69 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  70 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  71 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.opendaylight.yangtools.yang.common.Uint32;
 122 import org.opendaylight.yangtools.yang.common.Uint64;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 @Singleton
 127 public class VpnInterfaceManager extends AbstractAsyncDataTreeChangeListener<VpnInterface> {
 128
 129     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 130     private static final short DJC_MAX_RETRIES = 3;
 131
 132     private final DataBroker dataBroker;
 133     private final ManagedNewTransactionRunner txRunner;
 134     private final IBgpManager bgpManager;
 135     private final IFibManager fibManager;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IdManagerService idManager;
 138     private final OdlInterfaceRpcService ifaceMgrRpcService;
 139     private final VpnFootprintService vpnFootprintService;
 140     private final IInterfaceManager interfaceManager;
 141     private final IVpnManager vpnManager;
 142     private final ArpResponderHandler arpResponderHandler;
 143     private final JobCoordinator jobCoordinator;
 144     private final VpnUtil vpnUtil;
 145
 146     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 147
 148     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 149             new ConcurrentHashMap<>();
 150
 151     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 152
 153     @Inject
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final VpnFootprintService vpnFootprintService,
 161                                final IInterfaceManager interfaceManager,
 162                                final IVpnManager vpnManager,
 163                                final ArpResponderHandler arpResponderHandler,
 164                                final JobCoordinator jobCoordinator,
 165                                final CacheProvider cacheProvider,
 166                                final VpnUtil vpnUtil) {
 167         super(dataBroker, LogicalDatastoreType.CONFIGURATION,
 168                 InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class),
 169                 Executors.newListeningSingleThreadExecutor("VpnInterfaceManager", LOG));
 170
 171         this.dataBroker = dataBroker;
 172         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 173         this.bgpManager = bgpManager;
 174         this.idManager = idManager;
 175         this.mdsalManager = mdsalManager;
 176         this.fibManager = fibManager;
 177         this.ifaceMgrRpcService = ifaceMgrRpcService;
 178         this.vpnFootprintService = vpnFootprintService;
 179         this.interfaceManager = interfaceManager;
 180         this.vpnManager = vpnManager;
 181         this.arpResponderHandler = arpResponderHandler;
 182         this.jobCoordinator = jobCoordinator;
 183         this.vpnUtil = vpnUtil;
 184
 185         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 186                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 187                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 188         start();
 189     }
 190
 191     public Runnable isNotifyTaskQueued(String intfName) {
 192         return vpnIntfMap.remove(intfName);
 193     }
 194
 195     public void start() {
 196         LOG.info("{} start", getClass().getSimpleName());
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204         Executors.shutdownAndAwaitTermination(getExecutorService());
 205     }
 206
 207     @Override
 208     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 209         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 210         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 211                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(
 212                         new ArrayList<VpnInstanceNames>(vpnInterface.getVpnInstanceNames().values())));
 213         addVpnInterface(identifier, vpnInterface, null, null);
 214     }
 215
 216     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 217                           final VpnInterface vpnInterface, String vpnName) {
 218         // FIXME: separate this out somehow?
 219         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 220         lock.lock();
 221         try {
 222             if (isVpnInstanceReady(vpnName)) {
 223                 return true;
 224             }
 225             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 226             return false;
 227         } finally {
 228             lock.unlock();
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) {
 237             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 238             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 239         }
 240     }
 241
 242     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 244         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 245         final String interfaceName = key.getName();
 246
 247         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 248             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 249                   vpnName, vpnInterface.getName());
 250             return;
 251         }
 252         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 253              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 254         List<Adjacency> copyOldAdjs = null;
 255         if (oldAdjs != null) {
 256             copyOldAdjs = new ArrayList<>();
 257             copyOldAdjs.addAll(oldAdjs);
 258         }
 259         List<Adjacency> copyNewAdjs = null;
 260         if (newAdjs != null) {
 261             copyNewAdjs = new ArrayList<>();
 262             copyNewAdjs.addAll(newAdjs);
 263         }
 264         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 265     }
 266
 267     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 268                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 269                     final @Nullable List<Adjacency> newAdjs,
 270                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 271         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 272         final String interfaceName = key.getName();
 273         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 274         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 275             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 276             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 277             if (interfaceState != null) {
 278                 try {
 279                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 280                     final int ifIndex = interfaceState.getIfIndex();
 281                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 282                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 283                         // (the inventory tx goes in last)
 284                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 285                         //set of prefix used, as entry in prefix-to-interface datastore
 286                         // is prerequisite for refresh Fib to avoid race condition leading to
 287                         // missing remote next hop in bucket actions on bgp-vpn delete
 288                         Set<String> prefixListForRefreshFib = new HashSet<>();
 289                         ListenableFuture<Void> confFuture =
 290                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 291                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 292                                     operTx -> futures.add(
 293                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 294                                             LOG.info(
 295                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 296                                                     + " on dpn {}",
 297                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 298                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 299                                                 confTx, operTx, invTx, interfaceState, vpnName,
 300                                                 prefixListForRefreshFib);
 301                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 302                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 303                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 304                                                 if (newAdjs != null) {
 305                                                     for (Adjacency adj : newAdjs) {
 306                                                         if (oldAdjs.contains(adj)) {
 307                                                             oldAdjs.remove(adj);
 308                                                         } else {
 309                                                             if (!isBgpVpnInternetVpn
 310                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 311                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 312                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 313                                                                     prefixListForRefreshFib);
 314                                                             }
 315                                                         }
 316                                                     }
 317                                                 }
 318                                                 for (Adjacency adj : oldAdjs) {
 319                                                     if (!isBgpVpnInternetVpn
 320                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 321                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 322                                                             operTx, confTx);
 323                                                     }
 324                                                 }
 325                                             }
 326                                         })))));
 327                         Futures.addCallback(confFuture,
 328                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 329                             MoreExecutors.directExecutor());
 330                         futures.add(confFuture);
 331                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 332                             MoreExecutors.directExecutor());
 333                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 334                             + " processed successfully", interfaceName, vpnName, dpnId);
 335                         return futures;
 336                     });
 337                 } catch (NumberFormatException | IllegalStateException e) {
 338                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 339                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 340                         vpnName, e);
 341                     return;
 342                 }
 343             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 344                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 345                     () -> {
 346                         ListenableFuture<Void> future =
 347                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 348                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 349                                     confTx, vpnName);
 350                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 351                                     vpnName, vpnInterface.getDpnId());
 352                             });
 353                         ListenableFutures.addErrorLogging(future, LOG,
 354                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 355                         return Collections.singletonList(future);
 356                     });
 357             } else {
 358                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 359                     + " is not available", interfaceName, vpnName);
 360             }
 361         } else {
 362             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 363                     + " as vpn is pending delete", interfaceName, vpnName,
 364                 vpnInterface.getDpnId());
 365         }
 366     }
 367
 368     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 369     // see comments below.
 370     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 371     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 372             final int lportTag, boolean isInterfaceUp,
 373             TypedWriteTransaction<Configuration> writeConfigTxn,
 374             TypedWriteTransaction<Operational> writeOperTxn,
 375             TypedReadWriteTransaction<Configuration> writeInvTxn,
 376             Interface interfaceState, final String vpnName,
 377             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 378         final String interfaceName = vpnInterface.getName();
 379         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 380                 vpnName);
 381         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 382         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 383         if (!isInterfaceUp) {
 384             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 385                      interfaceName, dpId, vpnName);
 386             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 387             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 388                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 389                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 390                         dpId);
 391                 return;
 392             }
 393
 394             boolean waitForVpnInterfaceOpRemoval = false;
 395             if (opVpnInterface != null) {
 396                 String opVpnName = opVpnInterface.getVpnInstanceName();
 397                 String primaryInterfaceIp = null;
 398                 if (Objects.equals(opVpnName, vpnName)) {
 399                     // Please check if the primary VRF Entry does not exist for VPNInterface
 400                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 401                     // back to back
 402                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 403                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 404                     if (adjs == null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 406                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 407                                 vpnName);
 408                         return;
 409                     }
 410                     for (Adjacency adj : adjs) {
 411                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 412                             primaryInterfaceIp = adj.getIpAddress();
 413                             break;
 414                         }
 415                     }
 416                     if (primaryInterfaceIp == null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 418                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 419                                 dpId, vpnName);
 420                         return;
 421                     }
 422                     // Get the rd of the vpn instance
 423                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 424                     if (vrf != null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 426                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 427                         return;
 428                     }
 429                     waitForVpnInterfaceOpRemoval = true;
 430                 } else {
 431                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 432                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 433                 }
 434             }
 435             if (!waitForVpnInterfaceOpRemoval) {
 436                 // Add the VPNInterface and quit
 437                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 438                         null/*ipAddressSourceValuePair*/,
 439                         true /* add */);
 440                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 441                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 442                 if (!isBgpVpnInternetVpn) {
 443                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 444                 }
 445                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 446                         dpId, vpnName);
 447                 if (interfaceManager.isExternalInterface(interfaceName)) {
 448                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 449                         NwConstants.ADD_FLOW);
 450                 }
 451                 return;
 452             }
 453
 454             // FIB didn't get a chance yet to clean up this VPNInterface
 455             // Let us give it a chance here !
 456             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 457                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 458             try {
 459                 Runnable notifyTask = new VpnNotifyTask();
 460                 synchronized (notifyTask) {
 461                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 462                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 463                     // published here while holding the lock on it so this path will hit the wait before notify can be
 464                     // invoked.
 465                     vpnIntfMap.put(interfaceName, notifyTask);
 466                     try {
 467                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 468                     } catch (InterruptedException e) {
 469                         // Ignored
 470                     }
 471                 }
 472             } finally {
 473                 vpnIntfMap.remove(interfaceName);
 474             }
 475
 476             if (opVpnInterface != null) {
 477                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 478                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 479                         dpId, vpnName);
 480                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 481                 return;
 482             }
 483             // VPNInterface got removed, proceed with Add
 484             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 485                     interfaceName, dpId, vpnName);
 486             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 487                     null/*ipAddressSourceValuePair*/,
 488                     true /* add */);
 489             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 490                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 491             if (!isBgpVpnInternetVpn) {
 492                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 493             }
 494             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 495                     + " FIB to clean up", interfaceName, dpId, vpnName);
 496             if (interfaceManager.isExternalInterface(interfaceName)) {
 497                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 498                                lportTag, NwConstants.ADD_FLOW);
 499             }
 500         } else {
 501             try {
 502                 // Interface is retained in the DPN, but its Link Up.
 503                 // Advertise prefixes again for this interface to BGP
 504                 InstanceIdentifier<VpnInterface> identifier =
 505                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 506                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 507                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 508                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 509                 // Perform similar operation as interface add event for extraroutes.
 510                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 511                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 512                         LogicalDatastoreType.CONFIGURATION, path);
 513                 if (!optAdjacencies.isPresent()) {
 514                     LOG.trace("No config adjacencyKeyAdjacencyMap present for vpninterface {}", vpnInterface);
 515                     return;
 516                 }
 517                 Map<AdjacencyKey, Adjacency> adjacencyKeyAdjacencyMap = optAdjacencies.get().nonnullAdjacency();
 518                 for (Adjacency adjacency : adjacencyKeyAdjacencyMap.values()) {
 519                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 520                         continue;
 521                     }
 522                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 523                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 524                         continue;
 525                     }
 526                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 527                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 528                 }
 529             } catch (InterruptedException | ExecutionException e) {
 530                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 531                         interfaceName, vpnName, primaryRd, dpId);
 532             }
 533         }
 534     }
 535
 536     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 537         int lportTag, int addOrRemove) {
 538         Uuid extNetworkId;
 539         try {
 540             // vpn instance of ext-net interface is the network-id
 541             extNetworkId = new Uuid(vpnName);
 542         } catch (IllegalArgumentException e) {
 543             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 544                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 545             return;
 546         }
 547
 548         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 549         if (routerIds == null || routerIds.isEmpty()) {
 550             LOG.info("processExternalVpnInterface: No router is associated with {}."
 551                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 552                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 553             return;
 554         }
 555
 556         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 557                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 558         for (Uuid routerId : routerIds) {
 559             String routerName = routerId.getValue();
 560             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 561             if (Objects.equals(primarySwitch, dpId)) {
 562                 Routers router = vpnUtil.getExternalRouter(routerName);
 563                 if (router != null) {
 564                     if (addOrRemove == NwConstants.ADD_FLOW) {
 565                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 566                                 VpnUtil.getIpsListFromExternalIps(new ArrayList<ExternalIps>(router
 567                                                 .getExternalIps().values())), router.getExtGwMacAddress(),
 568                                 dpId, interfaceName, lportTag);
 569                     } else {
 570                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 571                                 VpnUtil.getIpsListFromExternalIps(new ArrayList<ExternalIps>(router
 572                                         .getExternalIps().values())),
 573                                 dpId, interfaceName, lportTag);
 574                     }
 575                 } else {
 576                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 577                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 578                             interfaceName, dpId, vpnName);
 579                 }
 580             }
 581         }
 582     }
 583
 584     // TODO Clean up the exception handling
 585     @SuppressWarnings("checkstyle:IllegalCatch")
 586     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 587                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 588                                                   String vpnName, String interfaceName) {
 589         if (rd == null) {
 590             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 591                   interfaceName, dpnId, vpnName);
 592             return;
 593         } else {
 594             if (rd.equals(vpnName)) {
 595                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 596                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 597                 return;
 598             }
 599         }
 600         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 601                 interfaceName, dpnId, vpnName, rd);
 602
 603         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 604         if (nextHopIp == null) {
 605             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 606                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 607                     rd, vpnName);
 608             return;
 609         }
 610
 611         try {
 612             //Read NextHops
 613             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 614             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 615                     LogicalDatastoreType.OPERATIONAL, path);
 616             if (adjacencies.isPresent()) {
 617                 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.get().getAdjacency();
 618                 if (nextHopsMap != null && !nextHopsMap.isEmpty()) {
 619                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 620                             + " rd {}", nextHopsMap, interfaceName, dpnId, vpnName, rd);
 621                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 622                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 623                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 624                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 625                     for (Adjacency nextHop : nextHopsMap.values()) {
 626                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 627                             continue;
 628                         }
 629                         String gatewayMac = null;
 630                         Uint32 label = Uint32.ZERO;
 631                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 632                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 633                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 634                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 635                                     .get();
 636                         } else {
 637                             label = nextHop.getLabel();
 638                         }
 639                         try {
 640                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 641                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 642                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 643                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 644                                     gatewayMac);
 645                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 646                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 647                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 648                         } catch (Exception e) {
 649                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 650                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 651                                     interfaceName, dpnId, e);
 652                         }
 653                     }
 654                 }
 655             }
 656         } catch (InterruptedException | ExecutionException e) {
 657             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 658                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 659         }
 660     }
 661
 662     // TODO Clean up the exception handling
 663     @SuppressWarnings("checkstyle:IllegalCatch")
 664     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 665                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 666                         TypedWriteTransaction<Operational> writeOperTx) {
 667         //Read NextHops
 668         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 669         String rd = vpnUtil.getVpnRd(interfaceName);
 670         if (rd == null) {
 671             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 672                 interfaceName, vpnName);
 673             return;
 674         } else {
 675             if (rd.equals(vpnName)) {
 676                 LOG.info(
 677                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 678                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 679                 return;
 680             }
 681         }
 682         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 683                vpnName, rd);
 684         Optional<AdjacenciesOp> adjacencies = Optional.empty();
 685         try {
 686             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 687                     path);
 688         } catch (InterruptedException | ExecutionException e) {
 689             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 690                     interfaceName, vpnName);
 691         }
 692         if (adjacencies.isPresent()) {
 693             Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.get().getAdjacency();
 694
 695             if (nextHopsMap != null && !nextHopsMap.isEmpty()) {
 696                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 697                         nextHopsMap, interfaceName, vpnName, rd);
 698                 for (Adjacency nextHop : nextHopsMap.values()) {
 699                     try {
 700                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 701                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 702                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 703                                     interfaceName, vpnName);
 704                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 705                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 706                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 707                                     interfaceName, vpnName);
 708                         } else if (nextHop.getNextHopIpList() != null) {
 709                             // Perform similar operation as interface delete event for extraroutes.
 710                             String allocatedRd = nextHop.getVrfId();
 711                             for (String nh : nextHop.getNextHopIpList()) {
 712                                 deleteExtraRouteFromCurrentAndImportingVpns(
 713                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 714                                     writeOperTx);
 715                             }
 716                         }
 717                     } catch (Exception e) {
 718                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 719                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 720                     }
 721                 }
 722             }
 723         }
 724     }
 725
 726     @SuppressWarnings("checkstyle:IllegalCatch")
 727     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 728                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 729                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 730                                                   TypedWriteTransaction<Operational> writeOperTxn,
 731                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 732                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 733             throws ExecutionException, InterruptedException {
 734         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 735         // Read NextHops
 736         Optional<VpnInterface> vpnInteface = Optional.empty();
 737         try {
 738             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 739             LogicalDatastoreType.CONFIGURATION, identifier);
 740         } catch (InterruptedException | ExecutionException e) {
 741             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 742                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 743         }
 744         Uuid intfnetworkUuid = null;
 745         NetworkType networkType = null;
 746         Long segmentationId = Long.valueOf(-1);
 747         Adjacencies adjacencies = null;
 748         if (vpnInteface.isPresent()) {
 749             intfnetworkUuid = vpnInteface.get().getNetworkId();
 750             networkType = vpnInteface.get().getNetworkType();
 751             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 752             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 753             if (adjacencies == null) {
 754                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 755                         null/*gwMac*/,  null/*gatewayIp*/, writeOperTxn);
 756                 return;
 757             }
 758         }
 759         // Get the rd of the vpn instance
 760         String nextHopIp = null;
 761         String gatewayIp = null;
 762         try {
 763             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 764         } catch (Exception e) {
 765             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 766                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 767         }
 768         List<String> nhList = new ArrayList<>();
 769         if (nextHopIp != null) {
 770             nhList.add(nextHopIp);
 771             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 772                     interfaceName, dpnId, vpnName, nhList);
 773         }
 774         Optional<String> gwMac = Optional.empty();
 775         String vpnInterfaceSubnetGwMacAddress = null;
 776         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 777         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 778         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 779         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 780         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 781         Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies != null ? adjacencies.getAdjacency()
 782                 : Collections.<AdjacencyKey, Adjacency>emptyMap();
 783         List<Adjacency> value = new ArrayList<>();
 784         for (Adjacency nextHop : nextHopsMap.values()) {
 785             String rd = primaryRd;
 786             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 787             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 788                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 789                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 790                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 791                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 792                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 793                 continue;
 794             }
 795             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 796                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 797                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 798                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 799                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHopsMap {} "
 800                         + "on dpn {} for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 801
 802                 Prefixes prefixes = intfnetworkUuid != null
 803                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 804                             segmentationId, prefixCue) :
 805                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 806                 writeOperTxn.mergeParentStructureMerge(VpnUtil.getPrefixToInterfaceIdentifier(
 807                         vpnUtil.getVpnId(vpnName), prefix), prefixes);
 808                 final Uuid subnetId = nextHop.getSubnetId();
 809
 810                 gatewayIp = nextHop.getSubnetGatewayIp();
 811                 if (gatewayIp == null) {
 812                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 813                     if (gatewayIpOptional.isPresent()) {
 814                         gatewayIp = gatewayIpOptional.get();
 815                     }
 816                 }
 817
 818                 if (gatewayIp != null) {
 819                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 820                     if (gwMac.isPresent()) {
 821                         // A valid mac-address is available for this subnet-gateway-ip
 822                         // Use this for programming ARP_RESPONDER table here.  And save this
 823                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 824                         // to populate L3_GW_MAC_TABLE there.
 825                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 826                                 gatewayIp, gwMac.get());
 827                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 828                     } else {
 829                         // A valid mac-address is not available for this subnet-gateway-ip
 830                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 831                         // Save this connected-mac-address as gateway-mac-address for the
 832                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 833                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 834                         if (gwMac.isPresent()) {
 835                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 836                                     NwConstants.ADD_FLOW, gwMac.get());
 837                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 838                                     gatewayIp, gwMac.get());
 839                         } else {
 840                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 841                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 842                                 + "gwIp {}",
 843                                 subnetId, interfaceName, vpnName, gatewayIp);
 844                         }
 845                     }
 846                 } else {
 847                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 848                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 849                         subnetId, interfaceName, vpnName);
 850                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 851                 }
 852                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHopsMap {} on dpn {}"
 853                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 854             } else {
 855                 //Extra route adjacency
 856                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 857                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 858                 // FIXME: separate this out somehow?
 859                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 860                 lock.lock();
 861                 try {
 862                     java.util.Optional<String> rdToAllocate = vpnUtil
 863                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 864                                     nextHop.getNextHopIpList().get(0), dpnId);
 865                     if (rdToAllocate.isPresent()) {
 866                         rd = rdToAllocate.get();
 867                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 868                             rd, prefix);
 869                     } else {
 870                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 871                         continue;
 872                     }
 873                 } finally {
 874                     lock.unlock();
 875                 }
 876                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 877                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 878                         interfaceName, dpnId);
 879             }
 880             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 881             // can be different from the gateway-mac-address within the VRFEntry as the
 882             // gateway-mac-address is a superset.
 883             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 884             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 885                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 886                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 887             Adjacency operationalAdjacency = null;
 888             try {
 889                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 890             } catch (NullPointerException e) {
 891                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 892                     input, e.getMessage());
 893                 return;
 894             }
 895             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 896                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 897                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 898                     writeConfigTxn);
 899             }
 900             value.add(operationalAdjacency);
 901         }
 902
 903         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 904         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 905                 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
 906
 907         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 908                 .setGatewayMac(gwMac.orElse(null)).setInterfaceName(interfaceName)
 909                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 910
 911         for (Adjacency nextHop : aug.getAdjacency().values()) {
 912             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 913             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 914                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 915                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 916                 registeredPopulator.populateFib(input, writeConfigTxn);
 917             }
 918         }
 919     }
 920
 921     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 922                                               long lportTag, String gwMac, String gwIp,
 923                                               TypedWriteTransaction<Operational> writeOperTxn) {
 924         VpnInterfaceOpDataEntry opInterface =
 925               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
 926         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 927             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 928         writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
 929         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 930                 interfaceName, dpnId, vpnName);
 931     }
 932
 933     // TODO Clean up the exception handling
 934     @SuppressWarnings("checkstyle:IllegalCatch")
 935     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 936                                            StateTunnelList stateTunnelList,
 937                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 938                                            TypedWriteTransaction<Operational> writeOperTxn) {
 939
 940         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 941         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 942         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 943         Map<AdjacencyKey, Adjacency> keyAdjacencyMap =
 944             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency()
 945                     : Collections.<AdjacencyKey, Adjacency>emptyMap();
 946         if (keyAdjacencyMap.isEmpty()) {
 947             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 948                     vpnInterface, srcDpnId);
 949             return;
 950         }
 951         String prefix = null;
 952         List<Adjacency> value = new ArrayList<>();
 953         boolean isFibNextHopAddReqd = false;
 954         String vpnName = vpnInterface.getVpnInstanceName();
 955         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 956         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 957         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 958                 vpnInterface.getName(), vpnInterface.getDpnId(),
 959                 vpnInterface.getVpnInstanceName(), keyAdjacencyMap);
 960         for (Adjacency adj : keyAdjacencyMap.values()) {
 961             String rd = adj.getVrfId();
 962             rd = rd != null ? rd : vpnName;
 963             prefix = adj.getIpAddress();
 964             Uint32 label = adj.getLabel();
 965             List<String> nhList = Collections.singletonList(srcTepIp);
 966             List<String> nextHopList = adj.getNextHopIpList();
 967             // If TEP is added , update the nexthop of primary adjacency.
 968             // Secondary adj nexthop is already pointing to primary adj IP address.
 969             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 970                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 971                 if (nextHopList != null && !nextHopList.isEmpty()) {
 972                     /* everything right already */
 973                 } else {
 974                     isFibNextHopAddReqd = true;
 975                 }
 976             } else {
 977                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 978                 if (!vrfEntryOptional.isPresent()) {
 979                     continue;
 980                 }
 981                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 982                 if (!nhList.contains(srcTepIp)) {
 983                     nhList.add(srcTepIp);
 984                     isFibNextHopAddReqd = true;
 985                 }
 986                 value.add(adj);
 987             }
 988
 989             if (isFibNextHopAddReqd) {
 990                 updateLabelMapper(label, nhList);
 991                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 992                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 993                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 994                 // Update the VRF entry with nextHop
 995                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, true,
 996                         writeConfigTxn);
 997
 998                 //Get the list of VPN's importing this route(prefix) .
 999                 // Then update the VRF entry with nhList
1000                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1001                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1002                     String vpnRd = vpn.getVrfId();
1003                     if (vpnRd != null) {
1004                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, true,
1005                                 writeConfigTxn);
1006                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1007                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1008                             vpnInterface.getName(), srcDpnId, vpnName,
1009                             vpn.getVpnInstanceName(), vpnRd);
1010                     }
1011                 }
1012                 // Advertise the prefix to BGP only for external vpn
1013                 // since there is a nexthop change.
1014                 try {
1015                     if (!rd.equalsIgnoreCase(vpnName)) {
1016                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1017                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1018                                 null /*gatewayMacAddress*/);
1019                     }
1020                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1021                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1022                             srcDpnId, vpnName);
1023                 } catch (Exception ex) {
1024                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1025                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1026                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1027                 }
1028             }
1029         }
1030         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1031         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1032                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1033                 .addAugmentation(AdjacenciesOp.class, aug).build();
1034         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1035                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1036         writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
1037         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1038                 vpnInterface.getName(), srcDpnId, vpnName);
1039
1040     }
1041
1042     // TODO Clean up the exception handling
1043     @SuppressWarnings("checkstyle:IllegalCatch")
1044     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1045                                               StateTunnelList stateTunnelList,
1046                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1047                                               TypedWriteTransaction<Operational> writeOperTxn) {
1048
1049         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1050         List<Adjacency> adjList = adjacencies != null ? new ArrayList<Adjacency>(adjacencies.getAdjacency().values())
1051                 : new ArrayList<>();
1052         String prefix = null;
1053         boolean isNextHopRemoveReqd = false;
1054         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1055         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1056         String vpnName = vpnInterface.getVpnInstanceName();
1057         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1058         String primaryRd = vpnUtil.getVpnRd(vpnName);
1059         if (adjList != null) {
1060             List<Adjacency> value = new ArrayList<>();
1061             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1062                     vpnInterface.getName(), vpnInterface.getDpnId(),
1063                     vpnInterface.getVpnInstanceName(), adjList);
1064             for (Adjacency adj : adjList) {
1065                 List<String> nhList = new ArrayList<>();
1066                 String rd = adj.getVrfId();
1067                 rd = rd != null ? rd : vpnName;
1068                 prefix = adj.getIpAddress();
1069                 List<String> nextHopList = adj.getNextHopIpList();
1070                 Uint32 label = adj.getLabel();
1071                 if (nextHopList != null && !nextHopList.isEmpty()) {
1072                     isNextHopRemoveReqd = true;
1073                 }
1074                 // If TEP is deleted , remove the nexthop from primary adjacency.
1075                 // Secondary adj nexthop will continue to point to primary adj IP address.
1076                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1077                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1078                 } else {
1079                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1080                     if (!vrfEntryOptional.isPresent()) {
1081                         continue;
1082                     }
1083                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1084                     if (nhList.contains(srcTepIp)) {
1085                         nhList.remove(srcTepIp);
1086                         isNextHopRemoveReqd = true;
1087                     }
1088                     value.add(adj);
1089                 }
1090
1091                 if (isNextHopRemoveReqd) {
1092                     updateLabelMapper(label, nhList);
1093                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1094                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1095                             prefix, nhList, vpnName,
1096                             vpnId, rd, vpnInterface.getName());
1097                     // Update the VRF entry with removed nextHop
1098                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp, label, false,
1099                             writeConfigTxn);
1100
1101                     //Get the list of VPN's importing this route(prefix) .
1102                     // Then update the VRF entry with nhList
1103                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1104                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1105                         String vpnRd = vpn.getVrfId();
1106                         if (vpnRd != null) {
1107                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, false,
1108                                     writeConfigTxn);
1109                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1110                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1111                                     nhList, label, vpnInterface.getName(), srcDpnId,
1112                                     vpnName,
1113                                     vpn.getVpnInstanceName(), vpnRd);
1114                         }
1115                     }
1116
1117                     // Withdraw prefix from BGP only for external vpn.
1118                     try {
1119                         if (!rd.equalsIgnoreCase(vpnName)) {
1120                             bgpManager.withdrawPrefix(rd, prefix);
1121                         }
1122                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1123                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1124                                 vpnInterface.getName(), srcDpnId,
1125                                 vpnName);
1126                     } catch (Exception ex) {
1127                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1128                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1129                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1130                     }
1131                 }
1132             }
1133             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1134             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1135                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1136                     .addAugmentation(AdjacenciesOp.class, aug).build();
1137             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1138                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1139             writeOperTxn.mergeParentStructurePut(interfaceId, opInterface);
1140             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1141                          vpnInterface.getName(), srcDpnId, vpnName);
1142         }
1143     }
1144
1145     @SuppressWarnings("checkstyle:IllegalCatch")
1146     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1147         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1148         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry;
1149         String vpnRd = vpnUtil.getVpnRd(vpnName);
1150         try {
1151             VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1152             if (opDataEntry == null) {
1153                 LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}"
1154                         + " when check for vpns exporting the routes", vpnName, vpnRd);
1155                 return vpnsToExportRoute;
1156             }
1157             vpnInstanceOpDataEntry = opDataEntry;
1158         } catch (Exception re) {
1159             LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}"
1160                     + " to check for vpns exporting the routes", vpnName, vpnRd, re);
1161             return vpnsToExportRoute;
1162         }
1163         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1164             if (input.getVpnInstanceName() == null) {
1165                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1166                         input.getVrfId());
1167                 return false;
1168             }
1169             return !input.getVpnInstanceName().equals(vpnName);
1170         };
1171
1172         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1173             Iterable<String> commonRTs =
1174                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1175                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1176             return Iterators.size(commonRTs.iterator()) > 0;
1177         };
1178
1179         vpnsToExportRoute =
1180                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1181                         Collectors.toList());
1182         return vpnsToExportRoute;
1183     }
1184
1185     // TODO Clean up the exception handling
1186     @SuppressWarnings("checkstyle:IllegalCatch")
1187     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1188         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1189         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1190             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1191             if (vrfEntries != null) {
1192                 ListenableFutures.addErrorLogging(
1193                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1194                         for (VrfEntry vrfEntry : vrfEntries) {
1195                             try {
1196                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1197                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1198                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1199                                             + " is not a controller managed non intervpn link route. Ignoring.",
1200                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1201                                     continue;
1202                                 }
1203                                 String prefix = vrfEntry.getDestPrefix();
1204                                 String gwMac = vrfEntry.getGatewayMacAddress();
1205                                 vrfEntry.nonnullRoutePaths().values().forEach(routePath -> {
1206                                     String nh = routePath.getNexthopAddress();
1207                                     Uint32 label = routePath.getLabel();
1208                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1209                                         vrfEntry.getOrigin()))) {
1210                                         LOG.info(
1211                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1212                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1213                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1214                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1215                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1216                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1217                                             confTx);
1218                                     } else {
1219                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1220                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1221                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1222                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1223                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1224                                             confTx);
1225                                     }
1226                                 });
1227                             } catch (RuntimeException e) {
1228                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1229                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1230                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1231                             }
1232                         }
1233                     }), LOG, "Error handing VPN exporting routes");
1234             } else {
1235                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1236                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1237             }
1238         }
1239     }
1240
1241     @Override
1242     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1243         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1244         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1245         final String interfaceName = key.getName();
1246         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) {
1247             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1248             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1249         }
1250     }
1251
1252     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1253                                 final VpnInterface vpnInterface, final String vpnName,
1254                                 final String interfaceName) {
1255         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1256             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1257                 ListenableFuture<Void> future =
1258                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1259                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1260                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1261                     });
1262                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1263                         vpnInterface.getName(), vpnName);
1264                 return Collections.singletonList(future);
1265             }, DJC_MAX_RETRIES);
1266         } else {
1267             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1268             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1269         }
1270     }
1271
1272     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1273     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1274                                            final VpnInterface vpnInterface, final String vpnName,
1275                                            final String interfaceName, final Interface interfaceState) {
1276         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1277                 vpnName, vpnInterface.getDpnId());
1278         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1279         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1280             () -> {
1281                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1282                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1283                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1284                         writeOperTxn -> futures.add(
1285                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1286                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1287                                         interfaceName, vpnName);
1288                                 Uint64 dpId;
1289                                 int ifIndex;
1290                                 String gwMacAddress;
1291                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1292                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1293                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1294                                 try {
1295                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1296                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1297                                 } catch (InterruptedException | ExecutionException e) {
1298                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1299                                             interfaceName, vpnName);
1300                                     return;
1301                                 }
1302                                 if (interfaceState != null) {
1303                                     try {
1304                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1305                                     } catch (NumberFormatException | IllegalStateException e) {
1306                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1307                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1308                                                         + " from vpn interface op data store. ", interfaceName,
1309                                                 vpnInterface.getDpnId(), vpnName, e);
1310                                         dpId = Uint64.ZERO;
1311                                     }
1312                                     ifIndex = interfaceState.getIfIndex();
1313                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1314                                 } else {
1315                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1316                                             + " from vpn interface op.", interfaceName);
1317                                     if (optVpnInterface.isPresent()) {
1318                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1319                                         dpId = vpnOpInterface.getDpnId();
1320                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1321                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1322                                     } else {
1323                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1324                                                 + " as interfaceState and vpn interface op is not"
1325                                                 + " available", interfaceName, vpnName);
1326                                         return;
1327                                     }
1328                                 }
1329                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1330                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1331                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1332                                 LOG.info(
1333                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1334                                                 + "successfully",
1335                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1336                             })))));
1337                 futures.add(configFuture);
1338                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1339                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1340                 return futures;
1341             }, DJC_MAX_RETRIES);
1342     }
1343
1344     protected void processVpnInterfaceDown(Uint64 dpId,
1345                                            String interfaceName,
1346                                            int lportTag,
1347                                            String gwMac,
1348                                            VpnInterfaceOpDataEntry vpnOpInterface,
1349                                            boolean isInterfaceStateDown,
1350                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1351                                            TypedWriteTransaction<Operational> writeOperTxn,
1352                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1353             throws ExecutionException, InterruptedException {
1354         if (vpnOpInterface == null) {
1355             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1356                     + " as it is not available in operational data store", interfaceName, dpId);
1357             return;
1358         }
1359         final String vpnName = vpnOpInterface.getVpnInstanceName();
1360         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1361                                                     interfaceName, vpnName);
1362         if (!isInterfaceStateDown) {
1363             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1364             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1365             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1366             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1367                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1368             if (interfaceManager.isExternalInterface(interfaceName)) {
1369                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1370                     NwConstants.DEL_FLOW);
1371             }
1372             if (!isBgpVpnInternetVpn) {
1373                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1374             }
1375             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1376                     + " successful", interfaceName, dpId, vpnName);
1377         } else {
1378             // Interface is retained in the DPN, but its Link Down.
1379             // Only withdraw the prefixes for this interface from BGP
1380             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1381         }
1382     }
1383
1384     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1385                                           final String vpnName, final Uint32 vpnId, String gwMac,
1386                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1387                                           TypedWriteTransaction<Operational> writeOperTxn,
1388                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1389             throws ExecutionException, InterruptedException {
1390         //Read NextHops
1391         try {
1392             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1393                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1394             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1395                     SingleTransactionDataBroker.syncReadOptional(dataBroker,
1396                             LogicalDatastoreType.OPERATIONAL, identifier);
1397             boolean isNonPrimaryAdjIp = false;
1398             String primaryRd = vpnUtil.getVpnRd(vpnName);
1399             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1400                     interfaceName, dpnId, vpnName, primaryRd);
1401             if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1402                 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1403                         + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1404                 return;
1405             }
1406             AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1407
1408             if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) {
1409                 Map<AdjacencyKey, Adjacency> nextHopsMap = adjacencies.getAdjacency();
1410                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1411                         interfaceName, dpnId, vpnName, nextHopsMap);
1412                 for (Adjacency nextHop : nextHopsMap.values()) {
1413                     if (nextHop.isPhysNetworkFunc()) {
1414                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1415                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1416                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1417                                 null, null/*writeCfgTxn*/);
1418                     } else {
1419                         String rd = nextHop.getVrfId();
1420                         List<String> nhList;
1421                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1422                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1423                             isNonPrimaryAdjIp = Boolean.TRUE;
1424                         } else {
1425                             // This is a primary adjacency
1426                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1427                                     : emptyList();
1428                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1429                                     vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1430                                     interfaceName, writeInvTxn);
1431                             isNonPrimaryAdjIp = Boolean.FALSE;
1432                         }
1433                         if (!nhList.isEmpty()) {
1434                             if (Objects.equals(primaryRd, vpnName)) {
1435                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1436                                 //remove from FIB directly
1437                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1438                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1439                             } else {
1440                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1441                                         interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn);
1442                             }
1443                         } else {
1444                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1445                                             + " interface {}", nextHop.getIpAddress(), rd,
1446                                     nextHop.getAdjacencyType().toString(), interfaceName);
1447                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1448                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1449                         }
1450                     }
1451                     String ip = nextHop.getIpAddress().split("/")[0];
1452                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1453                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1454                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1455                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1456                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1457                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1458                     }
1459                     // Remove the MIP-IP from VpnPortIpToPort.
1460                     if (isNonPrimaryAdjIp) {
1461                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1462                         if (persistedIp != null && persistedIp.isLearntIp()
1463                                 && persistedIp.getPortName().equals(interfaceName)) {
1464                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1465                             LOG.info(
1466                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1467                                             + "from VpnPortipToPort",
1468                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1469                         }
1470                     }
1471                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1472                     if (vpnPortipToPort != null) {
1473                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1474                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1475                                  + "Interface {} ip {} on dpn {} for vpn {}",
1476                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1477                     }
1478                 }
1479             } else {
1480                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1481                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1482                         + " Removing it.", interfaceName, vpnName, dpnId);
1483                 writeOperTxn.delete(identifier);
1484             }
1485         } catch (InterruptedException | ExecutionException e) {
1486             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1487                     interfaceName, dpnId, vpnName);
1488         }
1489     }
1490
1491     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1492                                                             String interfaceName, Uint64 dpnId,
1493                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1494                                                             TypedWriteTransaction<Operational> writeOperTx) {
1495         return (nh) -> {
1496             String primaryRd = vpnUtil.getVpnRd(vpnName);
1497             String prefix = nextHop.getIpAddress();
1498             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1499             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1500                     nextHop, vpnName, interfaceName, dpnId);
1501             // FIXME: separate this out somehow?
1502             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1503             lock.lock();
1504             try {
1505                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1506                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1507                     //If extra-route is present behind at least one VM, then do not remove or update
1508                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1509                     //prefixtointerface DS
1510                     return;
1511                 }
1512                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1513                         writeConfigTxn);
1514             } finally {
1515                 lock.unlock();
1516             }
1517             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1518                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1519                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1520         };
1521     }
1522
1523     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1524                                            Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp,
1525                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1526                                            TypedWriteTransaction<Operational> writeOperTx) {
1527         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1528                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1529         nhList.forEach((nh) -> {
1530             //IRT: remove routes from other vpns importing it
1531             if (isNonPrimaryAdjIp) {
1532                 removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1533             } else {
1534                 vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1535                         nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1536             }
1537             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1538                 String vpnRd = vpn.getVrfId();
1539                 if (vpnRd != null) {
1540                     fibManager.removeOrUpdateFibEntry(vpnRd,
1541                             nextHop.getIpAddress(), nh, writeConfigTxn);
1542                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1543                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1544                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1545                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1546                 }
1547             }
1548         });
1549     }
1550
1551     @SuppressWarnings("checkstyle:IllegalCatch")
1552     private void removeLearntPrefixFromBGP(String rd, String prefix, String nextHop,
1553                                            TypedWriteTransaction<Configuration> writeConfigTxn) {
1554         try {
1555             if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) {
1556                 LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this"
1557                         + " operation", prefix, nextHop, rd);
1558                 return;
1559             }
1560             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}",
1561                     rd, prefix, nextHop);
1562             fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn);
1563             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1564             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}",
1565                     rd, prefix, nextHop);
1566         } catch (Exception e) {
1567             LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e);
1568         }
1569     }
1570
1571     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1572                                                  int lportTag, String gwMac, String gwIp, String interfaceName,
1573                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1574             throws ExecutionException, InterruptedException {
1575         final Uuid subnetId = nextHop.getSubnetId();
1576         if (nextHop.getSubnetGatewayMacAddress() == null) {
1577             // A valid mac-address was not available for this subnet-gateway-ip
1578             // So a connected-mac-address was used for this subnet and we need
1579             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1580             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1581         }
1582         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1583                 subnetId);
1584     }
1585
1586     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1587                                                   String interfaceName) {
1588         // This is either an extra-route (or) a learned IP via subnet-route
1589         List<String> nhList = null;
1590         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1591         if (nextHopIp == null || nextHopIp.isEmpty()) {
1592             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1593                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1594                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1595                     vpnName);
1596             nhList = emptyList();
1597         } else {
1598             nhList = Collections.singletonList(nextHopIp);
1599         }
1600         return nhList;
1601     }
1602
1603     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1604         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1605         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1606         // else use connected interface
1607         if (gwPort != null && gwPort.isSubnetIp()) {
1608             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1609                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1610             return Optional.of(gwPort.getMacAddress());
1611         }
1612         return Optional.empty();
1613     }
1614
1615     @Override
1616     public void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1617         final VpnInterface update) {
1618         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1619         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1620                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1621         if (original.equals(update)) {
1622             LOG.info("update: original {} update {} are same. No update required.", original, update);
1623             return;
1624         }
1625         final String vpnInterfaceName = update.getName();
1626         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1627         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1628         //handles switching between <internal VPN - external VPN>
1629         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1630             List<ListenableFuture<Void>> futures = new ArrayList<>();
1631             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1632                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1633                                 + "to newVpn(s) {}",
1634                         original.getName(), dpnId,
1635                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(
1636                                 new ArrayList<VpnInstanceNames>(original.getVpnInstanceNames().values())),
1637                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(
1638                                 new ArrayList<VpnInstanceNames>(update.getVpnInstanceNames().values())));
1639                 return emptyList();
1640             }
1641             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1642             return futures;
1643         });
1644     }
1645
1646     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1647                                                            VpnInterface original, VpnInterface update,
1648                                                            List<ListenableFuture<Void>> futures) {
1649         boolean isVpnInstanceUpdate = false;
1650         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1651         final String interfaceName = key.getName();
1652         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1653         List<String> oldVpnListCopy = new ArrayList<>();
1654         oldVpnListCopy.addAll(oldVpnList);
1655         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1656         List<String> newVpnListCopy = new ArrayList<>();
1657         newVpnListCopy.addAll(newVpnList);
1658
1659         oldVpnList.removeAll(newVpnList);
1660         newVpnList.removeAll(oldVpnListCopy);
1661         //This block will execute only on if there is a change in the VPN Instance.
1662         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1663             /*
1664              * Internet BGP-VPN Instance update with single router:
1665              * ====================================================
1666              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1667              *     1st VPN Instance : router VPN or external BGP-VPN.
1668              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1669              *
1670              * VPN Instance UPDATE:
1671              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1672              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1673              *
1674              * External BGP-VPN Instance update with single router:
1675              * ====================================================
1676              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1677              *
1678              * Updated VPN Instance will be always either internal router VPN to
1679              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1680              *
1681              * VPN Instance UPDATE:
1682              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1683              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1684              *
1685              * Dual Router VPN Instance Update:
1686              * ================================
1687              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1688              *
1689              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1690              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1691              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1692              *
1693              * Dual Router --> Associated with common external BGP-VPN Instance.
1694              * 1st router and 2nd router are getting associated with single External BGP-VPN
1695              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1696              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1697              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1698              *
1699              *
1700              */
1701             isVpnInstanceUpdate = true;
1702             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1703                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1704                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1705                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1706                     List<String> externalBgpVpnList = new ArrayList<>();
1707                     for (String newVpnName : newVpnListCopy) {
1708                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1709                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1710                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1711                                 .BgpvpnType.BGPVPNExternal) {
1712                             externalBgpVpnList.add(newVpnName);
1713                             break;
1714                         }
1715                     }
1716                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1717                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1718                             externalBgpVpnList, oldVpnListCopy, futures);
1719
1720                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1721                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1722                     //Identify the router VPN Instance and pass that value as oldVpnList
1723                     List<String> routerVpnList = new ArrayList<>();
1724                     for (String newVpnName : newVpnListCopy) {
1725                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1726                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1727                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1728                                 .BgpvpnType.VPN) {
1729                             routerVpnList.add(newVpnName);
1730                             break;
1731                         }
1732                     }
1733                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1734                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1735                             newVpnList, oldVpnListCopy, futures);
1736
1737                 } else {
1738                     //Handle remaining use cases.
1739                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1740                             oldVpnListCopy, futures);
1741                 }
1742             } else {
1743                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1744                         oldVpnListCopy, futures);
1745             }
1746         }
1747         return isVpnInstanceUpdate;
1748     }
1749
1750     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1751                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1752                                          List<String> newVpnList, List<String> oldVpnListCopy,
1753                                          List<ListenableFuture<Void>> futures) {
1754         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1755         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1756                 ? new ArrayList<Adjacency>(origAdjs.getAdjacency().values()) : new ArrayList<>();
1757         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1758         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1759                 ? new ArrayList<Adjacency>(updateAdjs.getAdjacency().values()) : new ArrayList<>();
1760
1761         boolean isOldVpnRemoveCallExecuted = false;
1762         for (String oldVpnName : oldVpnList) {
1763             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1764                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1765             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1766             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1767                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1768                     newVpnList);
1769             isOldVpnRemoveCallExecuted = true;
1770         }
1771         //Wait for previous interface bindings to be removed
1772         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1773             try {
1774                 Thread.sleep(2000);
1775             } catch (InterruptedException e) {
1776                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1777             }
1778         }
1779         for (String newVpnName : newVpnList) {
1780             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1781             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1782                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1783                         + "onto vpnName {} ", interfaceName, newVpnName);
1784                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1785                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1786                                 + "from oldVpn(s) {} to newVpn {} ",
1787                         interfaceName, oldVpnListCopy, newVpnName);
1788                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1789                  * Use Case:
1790                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1791                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1792                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1793                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1794                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1795                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1796                  *     router VPN as well as Internet BGP-VPN.
1797                  *
1798                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1799                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1800                  *                                       update in existing router VPN instance.
1801                  */
1802                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1803                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1804                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1805                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1806                 }
1807             } else {
1808                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1809                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1810                         interfaceName, oldVpnListCopy, newVpnName);
1811             }
1812         }
1813     }
1814
1815     // TODO Clean up the exception handling
1816     @SuppressWarnings("checkstyle:IllegalCatch")
1817     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1818                                                                     String vpnInterfaceName,
1819                                                                     List<ListenableFuture<Void>> futures) {
1820         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1821         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1822                 != null ? new ArrayList<Adjacency>(origAdjs.getAdjacency().values()) : new ArrayList<>();
1823         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1824         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1825                 != null ? new ArrayList<Adjacency>(updateAdjs.getAdjacency().values()) : new ArrayList<>();
1826
1827         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1828         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames().values()) {
1829             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1830             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1831             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1832             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1833             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1834                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1835                 //set of prefix used as entry in prefix-to-interface datastore
1836                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1837                 // in bucket actions on bgp-vpn delete
1838                 Set<String> prefixListForRefreshFib = new HashSet<>();
1839                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1840                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1841                         operTx -> {
1842                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1843                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1844                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1845                                     update.getName(), newVpnName);
1846                             //handle both addition and removal of adjacencies
1847                             // currently, new adjacency may be an extra route
1848                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1849                             if (!oldAdjs.equals(newAdjs)) {
1850                                 for (Adjacency adj : copyNewAdjs) {
1851                                     if (copyOldAdjs.contains(adj)) {
1852                                         copyOldAdjs.remove(adj);
1853                                     } else {
1854                                         // add new adjacency
1855                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1856                                             try {
1857                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1858                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1859                                             } catch (RuntimeException e) {
1860                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1861                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1862                                             }
1863                                         }
1864                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1865                                             + " added to vpn interface {} on vpn {} dpnId {}",
1866                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1867                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1868                                     }
1869                                 }
1870                                 for (Adjacency adj : copyOldAdjs) {
1871                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1872                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1873                                             && !adj.isPhysNetworkFunc()) {
1874                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1875                                                 confTx);
1876                                             //remove FIB entry
1877                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1878                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1879                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1880                                             //remove BGP entry
1881                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1882                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1883                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1884                                             }
1885                                         } else {
1886                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1887                                                 operTx, confTx);
1888                                         }
1889                                     }
1890                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1891                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1892                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1893                                 }
1894                             }
1895                         })));
1896                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1897                     MoreExecutors.directExecutor());
1898                 futures.add(configTxFuture);
1899                 for (ListenableFuture<Void> future : futures) {
1900                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1901                             update.getName(), update.getVpnInstanceNames());
1902                 }
1903             } else {
1904                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1905                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1906             }
1907         }
1908         return futures;
1909     }
1910
1911     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1912         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1913                 .toString();
1914         // FIXME: separate this out somehow?
1915         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1916         lock.lock();
1917         try {
1918             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1919                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1920             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1921                 LogicalDatastoreType.OPERATIONAL, lriIid);
1922             if (opResult.isPresent()) {
1923                 LabelRouteInfo labelRouteInfo =
1924                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1925                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1926                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1927             }
1928             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1929                     nextHopIpList);
1930         } catch (InterruptedException | ExecutionException e) {
1931             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1932                     nextHopIpList);
1933         } catch (TransactionCommitFailedException e) {
1934             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1935                     nextHopIpList);
1936         } finally {
1937             lock.unlock();
1938         }
1939     }
1940
1941     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1942         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1943
1944         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1945         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1946                 .addAugmentation(SubnetRoute.class, route).build();
1947         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1948         InstanceIdentifierBuilder<VrfTables> idBuilder =
1949             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1950         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1951         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1952         if (writeConfigTxn != null) {
1953             writeConfigTxn.mergeParentStructureMerge(vrfTableId, vrfTableNew);
1954         } else {
1955             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1956         }
1957         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1958                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1959     }
1960
1961     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1962                                            Adjacency adj, Uint64 dpnId,
1963                                            TypedWriteTransaction<Operational> writeOperTxn,
1964                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1965                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1966                                            Set<String> prefixListForRefreshFib)
1967             throws ExecutionException, InterruptedException {
1968         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1969         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1970         try {
1971             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1972                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1973             if (optVpnInterface.isPresent()) {
1974                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1975                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1976                 String vpnName = currVpnIntf.getVpnInstanceName();
1977                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1978                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1979                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1980                         LogicalDatastoreType.OPERATIONAL, adjPath);
1981                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1982                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1983                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1984                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1985                 List<Adjacency> adjacencies = new ArrayList<>();
1986                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1987                     adjacencies.addAll(optAdjacencies.get().getAdjacency().values());
1988                 }
1989                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1990                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1991                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1992                 Adjacency operationalAdjacency = null;
1993                 //Handling dual stack neutron port primary adjacency
1994                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1995                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1996                             currVpnIntf.getName(), vpnName);
1997                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1998                             currVpnIntf.getName());
1999                     if (interfaceState != null) {
2000                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
2001                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
2002                             prefixListForRefreshFib);
2003                     }
2004                 }
2005                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
2006                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
2007                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
2008                             : RouteOrigin.STATIC;
2009                     String nh = adj.getNextHopIpList().get(0);
2010                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
2011                     // FIXME: separate out to somehow?
2012                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
2013                     lock.lock();
2014                     try {
2015                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
2016                                 vpnId, null, prefix, vpnName, nh, dpnId);
2017                         if (rdToAllocate.isPresent()) {
2018                             input.setRd(rdToAllocate.get());
2019                             operationalAdjacency = populator.createOperationalAdjacency(input);
2020                             int label = operationalAdjacency.getLabel().intValue();
2021                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
2022                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
2023                                     currVpnIntf.getName(), operationalAdjacency, encapType,
2024                                     prefixListForRefreshFib, writeConfigTxn);
2025                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
2026                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
2027                                     vpnName, label, currVpnIntf.getName(), dpnId);
2028                         } else {
2029                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2030                                     vpnName, prefix);
2031                             return;
2032                         }
2033                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2034                         // Keeping the MPLS check for now.
2035                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2036                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2037                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2038                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
2039                             vpnsToImportRoute.forEach(vpn -> {
2040                                 if (vpn.getVrfId() != null) {
2041                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2042                                             vpnId, prefix,
2043                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2044                                             .ifPresent(
2045                                                 rds -> vpnManager.addExtraRoute(
2046                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2047                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2048                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2049                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2050                                 }
2051                             });
2052                         }
2053                     } finally {
2054                         lock.unlock();
2055                     }
2056                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2057                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2058                             currVpnIntf.getName(), vpnName);
2059
2060                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2061                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2062                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2063                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2064                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2065                             Prefixes.PrefixCue.PhysNetFunc);
2066                     if (vpnIntefaceConfig.isPresent()) {
2067                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2068                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2069                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2070                     }
2071
2072                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2073
2074                     writeOperTxn.mergeParentStructureMerge(
2075                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2076                                     prefix), pnfPrefix);
2077
2078                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2079                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2080                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2081                             RouteOrigin.LOCAL, writeConfigTxn);
2082
2083                     input.setRd(adj.getVrfId());
2084                 }
2085                 if (operationalAdjacency == null) {
2086                     operationalAdjacency = populator.createOperationalAdjacency(input);
2087                 }
2088                 adjacencies.add(operationalAdjacency);
2089                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2090                 VpnInterfaceOpDataEntry newVpnIntf =
2091                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2092                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2093                                 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2094                 writeOperTxn.mergeParentStructureMerge(identifier, newVpnIntf);
2095             }
2096         } catch (InterruptedException | ExecutionException e) {
2097             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2098                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2099         }
2100     }
2101
2102     @Nullable
2103     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2104         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2105         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2106     }
2107
2108     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2109                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2110                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2111         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2112         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2113         try {
2114             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2115                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2116             if (optVpnInterface.isPresent()) {
2117                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2118                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2119                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2120                         LogicalDatastoreType.OPERATIONAL, path);
2121                 if (optAdjacencies.isPresent()) {
2122                     Map<AdjacencyKey, Adjacency> keyAdjacencyMap = optAdjacencies.get().getAdjacency();
2123
2124                     if (keyAdjacencyMap != null && !keyAdjacencyMap.isEmpty()) {
2125                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", keyAdjacencyMap);
2126                         for (Adjacency adjacency : keyAdjacencyMap.values()) {
2127                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2128                                 String rd = adjacency.getVrfId();
2129                                 if (adj.getNextHopIpList() != null) {
2130                                     for (String nh : adj.getNextHopIpList()) {
2131                                         deleteExtraRouteFromCurrentAndImportingVpns(
2132                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2133                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2134                                     }
2135                                 } else if (adj.isPhysNetworkFunc()) {
2136                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2137                                             adj.getIpAddress(), adj.getSubnetId());
2138                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2139                                             null, writeConfigTxn);
2140                                 }
2141                                 break;
2142                             }
2143
2144                         }
2145                     }
2146                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2147                             dpnId, adj.getVrfId());
2148                 } else {
2149                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2150                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2151                 }
2152             }
2153         } catch (InterruptedException | ExecutionException e) {
2154             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2155                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2156         }
2157     }
2158
2159     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2160                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2161                                     TypedWriteTransaction<Operational> writeOperTx) {
2162         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2163                 destination, nextHop, vpnName, intfName, rd);
2164         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2165         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2166         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2167             String vpnRd = vpn.getVrfId();
2168             if (vpnRd != null) {
2169                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2170                         destination, nextHop, vpnName, intfName, vpnRd);
2171                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2172                         writeOperTx);
2173             }
2174         }
2175     }
2176
2177     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2178         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2179             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2180             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2181     }
2182
2183     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2184         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2185             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2186     }
2187
2188     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2189                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2190         if (vpnInterface == null) {
2191             return;
2192         }
2193         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2194         if (adjs == null) {
2195             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2196                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2197             return;
2198         }
2199         for (Adjacency adj : adjs) {
2200             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2201                 String primaryInterfaceIp = adj.getIpAddress();
2202                 String macAddress = adj.getMacAddress();
2203                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2204
2205                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2206                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2207                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
2208                     LOG.error(
2209                             "createFibEntryForRouterInterface: Unable to retrieve label for vpn pool {}, "
2210                                     + "vpninterface {}, vpn {}, rd {}",
2211                             VpnConstants.VPN_IDPOOL_NAME, interfaceName, vpnName, primaryRd);
2212                     return;
2213                 }
2214                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2215                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2216                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2217                         routerInt, label, writeConfigTxn);
2218                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2219                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2220                         macAddress);
2221             } else {
2222                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2223                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2224                         interfaceName, primaryRd, vpnName);
2225             }
2226         }
2227     }
2228
2229     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2230             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2231         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2232         String rd = vpnUtil.getVpnRd(vpnName);
2233         if (adjs != null) {
2234             Map<AdjacencyKey, Adjacency> keyAdjacencyMap = adjs.nonnullAdjacency();
2235             for (Adjacency adj : keyAdjacencyMap.values()) {
2236                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2237                     String primaryInterfaceIp = adj.getIpAddress();
2238                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2239                     fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2240                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2241                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2242                 }
2243             }
2244         } else {
2245             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2246                     vpnInterface.getName(), rd);
2247         }
2248     }
2249
2250     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2251         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2252         final String interfaceName = key.getName();
2253         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2254                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2255         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2256                   intefaceData.identifier, vpnName);
2257     }
2258
2259     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2260                                               VpnInterface vpnInterface, String vpnName) {
2261         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2262                .get(vpnName);
2263         if (vpnInterfaces == null) {
2264             vpnInterfaces = new ConcurrentLinkedQueue<>();
2265         }
2266         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2267         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2268         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2269                 vpnInterface.getName(), vpnName);
2270     }
2271
2272     public boolean isVpnInstanceReady(String vpnInstanceName) {
2273         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2274         if (vpnRd == null) {
2275             return false;
2276         }
2277         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2278
2279         return vpnInstanceOpDataEntry != null;
2280     }
2281
2282     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2283         // FIXME: separate out to somehow?
2284         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2285         lock.lock();
2286         try {
2287             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2288                     unprocessedVpnInterfaces.get(vpnInstanceName);
2289             if (vpnInterfaces != null) {
2290                 while (!vpnInterfaces.isEmpty()) {
2291                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2292                     if (hasVpnInstanceCreatedSuccessfully) {
2293                         processSavedInterface(savedInterface, vpnInstanceName);
2294                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2295                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2296                     } else {
2297                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2298                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2299                     }
2300                 }
2301             } else {
2302                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2303             }
2304         } finally {
2305             lock.unlock();
2306         }
2307     }
2308
2309     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2310             VpnInterface vpnInterface) {
2311         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2312         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2313         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2314         lock.lock();
2315         try {
2316             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2317                     unprocessedVpnInterfaces.get(firstVpnName);
2318             if (vpnInterfaces != null) {
2319                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2320                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2321                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2322                 }
2323             } else {
2324                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2325             }
2326         } finally {
2327             lock.unlock();
2328         }
2329     }
2330
2331     public void vpnInstanceIsReady(String vpnInstanceName) {
2332         processSavedInterfaces(vpnInstanceName, true);
2333     }
2334
2335     public void vpnInstanceFailed(String vpnInstanceName) {
2336         processSavedInterfaces(vpnInstanceName, false);
2337     }
2338
2339     private static class UnprocessedVpnInterfaceData {
2340         InstanceIdentifier<VpnInterface> identifier;
2341         VpnInterface vpnInterface;
2342
2343         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2344             this.identifier = identifier;
2345             this.vpnInterface = vpnInterface;
2346         }
2347
2348         @Override
2349         public int hashCode() {
2350             final int prime = 31;
2351             int result = 1;
2352             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2353             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2354             return result;
2355         }
2356
2357         @Override
2358         public boolean equals(Object obj) {
2359             if (this == obj) {
2360                 return true;
2361             }
2362             if (obj == null) {
2363                 return false;
2364             }
2365             if (getClass() != obj.getClass()) {
2366                 return false;
2367             }
2368             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2369             if (identifier == null) {
2370                 if (other.identifier != null) {
2371                     return false;
2372                 }
2373             } else if (!identifier.equals(other.identifier)) {
2374                 return false;
2375             }
2376             if (vpnInterface == null) {
2377                 if (other.vpnInterface != null) {
2378                     return false;
2379                 }
2380             } else if (!vpnInterface.equals(other.vpnInterface)) {
2381                 return false;
2382             }
2383             return true;
2384         }
2385     }
2386
2387     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2388         String primaryRd = vpnUtil.getVpnRd(vpnName);
2389         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2390         if (vpnInstanceOpData == null) {
2391             return;
2392         }
2393         List<VpnToDpnList> vpnToDpnLists = new ArrayList<VpnToDpnList>(vpnInstanceOpData.getVpnToDpnList().values());
2394         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2395             return;
2396         }
2397         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2398         vpnToDpnLists.forEach(vpnToDpnList -> {
2399             if (vpnToDpnList.getVpnInterfaces() == null) {
2400                 return;
2401             }
2402             vpnToDpnList.getVpnInterfaces().values().forEach(vpnInterface -> {
2403                 try {
2404                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2405                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2406                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2407                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2408                     if (!vpnInterfaceOptional.isPresent()) {
2409                         return;
2410                     }
2411                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2412                             vpnInterface.getInterfaceName());
2413                     if (configVpnAdjacencies == null) {
2414                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2415                         return;
2416                     }
2417                     List<Adjacency> operationVpnAdjacencies = new ArrayList<Adjacency>(vpnInterfaceOptional.get()
2418                             .augmentation(AdjacenciesOp.class).nonnullAdjacency().values());
2419                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2420                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2421                     // in operational DS. These unprocessed adjacencies will be handled below.
2422                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2423                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2424                     configVpnAdjacencies.stream()
2425                         .filter(adjacency -> operationVpnAdjacencies.stream()
2426                                 .noneMatch(operationalAdjacency ->
2427                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2428                         .forEach(adjacency -> {
2429                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2430                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2431                                 () -> {
2432                                     // TODO Deal with sequencing — the config tx must only submitted
2433                                     // if the oper tx goes in
2434                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2435                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2436                                         futures.add(
2437                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2438                                                 //set of prefix used, as entry in prefix-to-interface datastore
2439                                                 // is prerequisite for refresh Fib to avoid race condition leading
2440                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2441                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2442                                                 ListenableFuture<Void> configTxFuture =
2443                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2444                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2445                                                             primaryRd, adjacency,
2446                                                             vpnInterfaceOptional.get().getDpnId(),
2447                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2448                                                 Futures.addCallback(configTxFuture,
2449                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2450                                                     MoreExecutors.directExecutor());
2451                                                 futures.add(configTxFuture);
2452                                             }));
2453                                         return futures;
2454                                     } else {
2455                                         return emptyList();
2456                                     }
2457                                 });
2458                         });
2459                 } catch (InterruptedException | ExecutionException e) {
2460                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2461                             vpnName, primaryRd);
2462                 }
2463             });
2464         });
2465     }
2466
2467     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2468         private final String interfaceName;
2469         private final boolean add;
2470         private final String txnDestination;
2471
2472         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2473             this.interfaceName = interfaceName;
2474             this.add = add;
2475             this.txnDestination = transactionDest;
2476         }
2477
2478         @Override
2479         public void onSuccess(Void voidObj) {
2480             if (add) {
2481                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2482                         interfaceName, txnDestination);
2483             } else {
2484                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2485             }
2486         }
2487
2488         @Override
2489         public void onFailure(Throwable throwable) {
2490             if (add) {
2491                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2492                         interfaceName, txnDestination, throwable);
2493             } else {
2494                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2495                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2496             }
2497         }
2498     }
2499
2500     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2501         private final String primaryRd;
2502         private final Set<String> prefixListForRefreshFib;
2503
2504         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2505             this.primaryRd = primaryRd;
2506             this.prefixListForRefreshFib = prefixListForRefreshFib;
2507         }
2508
2509         @Override
2510         public void onSuccess(Void voidObj) {
2511             prefixListForRefreshFib.forEach(prefix -> {
2512                 fibManager.refreshVrfEntry(primaryRd, prefix);
2513             });
2514         }
2515
2516         @Override
2517         public void onFailure(Throwable throwable) {
2518             LOG.debug("write Tx config operation failed", throwable);
2519         }
2520     }
2521 }