vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.util.ArrayList;
  25 import java.util.Collections;
  26 import java.util.HashSet;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.locks.ReentrantLock;
  35 import java.util.function.Consumer;
  36 import java.util.function.Predicate;
  37 import java.util.stream.Collectors;
  38 import javax.annotation.PostConstruct;
  39 import javax.annotation.PreDestroy;
  40 import javax.inject.Inject;
  41 import javax.inject.Singleton;
  42 import org.eclipse.jdt.annotation.Nullable;
  43 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  46 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  47 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  48 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  49 import org.opendaylight.genius.infra.Datastore.Configuration;
  50 import org.opendaylight.genius.infra.Datastore.Operational;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  52 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  53 import org.opendaylight.genius.infra.TransactionAdapter;
  54 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  55 import org.opendaylight.genius.infra.TypedWriteTransaction;
  56 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  57 import org.opendaylight.genius.mdsalutil.NWUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.genius.utils.JvmGlobalLocks;
  62 import org.opendaylight.infrautils.caches.CacheProvider;
  63 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  64 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  65 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  66 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  67 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  68 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  69 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  70 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  71 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  72 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  73 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  74 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  75 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  79 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 @Singleton
 127 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 128
 129     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 130     private static final short DJC_MAX_RETRIES = 3;
 131
 132     private final DataBroker dataBroker;
 133     private final ManagedNewTransactionRunner txRunner;
 134     private final IBgpManager bgpManager;
 135     private final IFibManager fibManager;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IdManagerService idManager;
 138     private final OdlInterfaceRpcService ifaceMgrRpcService;
 139     private final VpnFootprintService vpnFootprintService;
 140     private final IInterfaceManager interfaceManager;
 141     private final IVpnManager vpnManager;
 142     private final ArpResponderHandler arpResponderHandler;
 143     private final JobCoordinator jobCoordinator;
 144     private final VpnUtil vpnUtil;
 145
 146     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 147
 148     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 149             new ConcurrentHashMap<>();
 150
 151     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 152
 153     @Inject
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final VpnFootprintService vpnFootprintService,
 161                                final IInterfaceManager interfaceManager,
 162                                final IVpnManager vpnManager,
 163                                final ArpResponderHandler arpResponderHandler,
 164                                final JobCoordinator jobCoordinator,
 165                                final CacheProvider cacheProvider,
 166                                final VpnUtil vpnUtil) {
 167         super(VpnInterface.class, VpnInterfaceManager.class);
 168
 169         this.dataBroker = dataBroker;
 170         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 171         this.bgpManager = bgpManager;
 172         this.idManager = idManager;
 173         this.mdsalManager = mdsalManager;
 174         this.fibManager = fibManager;
 175         this.ifaceMgrRpcService = ifaceMgrRpcService;
 176         this.vpnFootprintService = vpnFootprintService;
 177         this.interfaceManager = interfaceManager;
 178         this.vpnManager = vpnManager;
 179         this.arpResponderHandler = arpResponderHandler;
 180         this.jobCoordinator = jobCoordinator;
 181         this.vpnUtil = vpnUtil;
 182
 183         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 184                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 185                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 186     }
 187
 188     public Runnable isNotifyTaskQueued(String intfName) {
 189         return vpnIntfMap.remove(intfName);
 190     }
 191
 192     @PostConstruct
 193     public void start() {
 194         LOG.info("{} start", getClass().getSimpleName());
 195         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 196     }
 197
 198     @Override
 199     @PreDestroy
 200     public void close() {
 201         super.close();
 202         vpnInstanceOpDataEntryCache.close();
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 207         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 208     }
 209
 210     @Override
 211     protected VpnInterfaceManager getDataTreeChangeListener() {
 212         return VpnInterfaceManager.this;
 213     }
 214
 215     @Override
 216     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 217         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 218         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 219                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 220         addVpnInterface(identifier, vpnInterface, null, null);
 221     }
 222
 223     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 224                           final VpnInterface vpnInterface, String vpnName) {
 225         // FIXME: separate this out somehow?
 226         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 227         lock.lock();
 228         try {
 229             if (isVpnInstanceReady(vpnName)) {
 230                 return true;
 231             }
 232             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 233             return false;
 234         } finally {
 235             lock.unlock();
 236         }
 237     }
 238
 239     // TODO Clean up the exception handling
 240     @SuppressWarnings("checkstyle:IllegalCatch")
 241     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 242                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 243         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 244             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 245             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 246         }
 247     }
 248
 249     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 250                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 251         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 252         final String interfaceName = key.getName();
 253
 254         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 255             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 256                   vpnName, vpnInterface.getName());
 257             return;
 258         }
 259         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 260              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 261         List<Adjacency> copyOldAdjs = null;
 262         if (oldAdjs != null) {
 263             copyOldAdjs = new ArrayList<>();
 264             copyOldAdjs.addAll(oldAdjs);
 265         }
 266         List<Adjacency> copyNewAdjs = null;
 267         if (newAdjs != null) {
 268             copyNewAdjs = new ArrayList<>();
 269             copyNewAdjs.addAll(newAdjs);
 270         }
 271         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 272     }
 273
 274     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 275                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 276                     final @Nullable List<Adjacency> newAdjs,
 277                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 278         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 279         final String interfaceName = key.getName();
 280         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 281         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 282             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 283             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 284             if (interfaceState != null) {
 285                 try {
 286                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 287                     final int ifIndex = interfaceState.getIfIndex();
 288                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 289                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 290                         // (the inventory tx goes in last)
 291                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 292                         //set of prefix used, as entry in prefix-to-interface datastore
 293                         // is prerequisite for refresh Fib to avoid race condition leading to
 294                         // missing remote next hop in bucket actions on bgp-vpn delete
 295                         Set<String> prefixListForRefreshFib = new HashSet<>();
 296                         ListenableFuture<Void> confFuture =
 297                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 298                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 299                                     operTx -> futures.add(
 300                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 301                                             LOG.info(
 302                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 303                                                     + " on dpn {}",
 304                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 305                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 306                                                 confTx, operTx, invTx, interfaceState, vpnName,
 307                                                 prefixListForRefreshFib);
 308                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 309                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 310                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 311                                                 if (newAdjs != null) {
 312                                                     for (Adjacency adj : newAdjs) {
 313                                                         if (oldAdjs.contains(adj)) {
 314                                                             oldAdjs.remove(adj);
 315                                                         } else {
 316                                                             if (!isBgpVpnInternetVpn
 317                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 318                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 319                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 320                                                                     prefixListForRefreshFib);
 321                                                             }
 322                                                         }
 323                                                     }
 324                                                 }
 325                                                 for (Adjacency adj : oldAdjs) {
 326                                                     if (!isBgpVpnInternetVpn
 327                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 328                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 329                                                             operTx, confTx);
 330                                                     }
 331                                                 }
 332                                             }
 333                                         })))));
 334                         Futures.addCallback(confFuture,
 335                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 336                             MoreExecutors.directExecutor());
 337                         futures.add(confFuture);
 338                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 339                             MoreExecutors.directExecutor());
 340                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 341                             + " processed successfully", interfaceName, vpnName, dpnId);
 342                         return futures;
 343                     });
 344                 } catch (NumberFormatException | IllegalStateException e) {
 345                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 346                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 347                         vpnName, e);
 348                     return;
 349                 }
 350             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 351                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 352                     () -> {
 353                         ListenableFuture<Void> future =
 354                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 355                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 356                                     confTx, vpnName);
 357                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 358                                     vpnName, vpnInterface.getDpnId());
 359                             });
 360                         ListenableFutures.addErrorLogging(future, LOG,
 361                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 362                         return Collections.singletonList(future);
 363                     });
 364             } else {
 365                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 366                     + " is not available", interfaceName, vpnName);
 367             }
 368         } else {
 369             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 370                     + " as vpn is pending delete", interfaceName, vpnName,
 371                 vpnInterface.getDpnId());
 372         }
 373     }
 374
 375     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 376     // see comments below.
 377     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 378     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 379             final int lportTag, boolean isInterfaceUp,
 380             TypedWriteTransaction<Configuration> writeConfigTxn,
 381             TypedWriteTransaction<Operational> writeOperTxn,
 382             TypedReadWriteTransaction<Configuration> writeInvTxn,
 383             Interface interfaceState, final String vpnName,
 384             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 385         final String interfaceName = vpnInterface.getName();
 386         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 387                 vpnName);
 388         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 389         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 390         if (!isInterfaceUp) {
 391             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 392                      interfaceName, dpId, vpnName);
 393             long vpnId = vpnUtil.getVpnId(vpnName);
 394             if (vpnId == VpnConstants.INVALID_ID) {
 395                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 396                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 397                         dpId);
 398                 return;
 399             }
 400
 401             boolean waitForVpnInterfaceOpRemoval = false;
 402             if (opVpnInterface != null) {
 403                 String opVpnName = opVpnInterface.getVpnInstanceName();
 404                 String primaryInterfaceIp = null;
 405                 if (Objects.equals(opVpnName, vpnName)) {
 406                     // Please check if the primary VRF Entry does not exist for VPNInterface
 407                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 408                     // back to back
 409                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 410                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 411                     if (adjs == null) {
 412                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 413                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 414                                 vpnName);
 415                         return;
 416                     }
 417                     for (Adjacency adj : adjs) {
 418                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 419                             primaryInterfaceIp = adj.getIpAddress();
 420                             break;
 421                         }
 422                     }
 423                     if (primaryInterfaceIp == null) {
 424                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 425                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 426                                 dpId, vpnName);
 427                         return;
 428                     }
 429                     // Get the rd of the vpn instance
 430                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 431                     if (vrf != null) {
 432                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 433                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 434                         return;
 435                     }
 436                     waitForVpnInterfaceOpRemoval = true;
 437                 } else {
 438                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 439                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 440                 }
 441             }
 442             if (!waitForVpnInterfaceOpRemoval) {
 443                 // Add the VPNInterface and quit
 444                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 445                         null/*ipAddressSourceValuePair*/,
 446                         true /* add */);
 447                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 448                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 449                 if (!isBgpVpnInternetVpn) {
 450                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 451                 }
 452                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 453                         dpId, vpnName);
 454                 if (interfaceManager.isExternalInterface(interfaceName)) {
 455                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 456                         NwConstants.ADD_FLOW);
 457                 }
 458                 return;
 459             }
 460
 461             // FIB didn't get a chance yet to clean up this VPNInterface
 462             // Let us give it a chance here !
 463             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 464                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 465             try {
 466                 Runnable notifyTask = new VpnNotifyTask();
 467                 synchronized (notifyTask) {
 468                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 469                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 470                     // published here while holding the lock on it so this path will hit the wait before notify can be
 471                     // invoked.
 472                     vpnIntfMap.put(interfaceName, notifyTask);
 473                     try {
 474                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 475                     } catch (InterruptedException e) {
 476                         // Ignored
 477                     }
 478                 }
 479             } finally {
 480                 vpnIntfMap.remove(interfaceName);
 481             }
 482
 483             if (opVpnInterface != null) {
 484                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 485                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 486                         dpId, vpnName);
 487                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 488                 return;
 489             }
 490             // VPNInterface got removed, proceed with Add
 491             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 492                     interfaceName, dpId, vpnName);
 493             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 494                     null/*ipAddressSourceValuePair*/,
 495                     true /* add */);
 496             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 497                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 498             if (!isBgpVpnInternetVpn) {
 499                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 500             }
 501             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 502                     + " FIB to clean up", interfaceName, dpId, vpnName);
 503             if (interfaceManager.isExternalInterface(interfaceName)) {
 504                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 505                                lportTag, NwConstants.ADD_FLOW);
 506             }
 507         } else {
 508             try {
 509                 // Interface is retained in the DPN, but its Link Up.
 510                 // Advertise prefixes again for this interface to BGP
 511                 InstanceIdentifier<VpnInterface> identifier =
 512                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 513                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 514                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 515                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 516                 // Perform similar operation as interface add event for extraroutes.
 517                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 518                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 519                         LogicalDatastoreType.CONFIGURATION, path);
 520                 if (!optAdjacencies.isPresent()) {
 521                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 522                     return;
 523                 }
 524                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 525                 for (Adjacency adjacency : adjacencies) {
 526                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 527                         continue;
 528                     }
 529                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 530                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 531                         continue;
 532                     }
 533                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 534                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 535                 }
 536             } catch (ReadFailedException e) {
 537                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 538                         interfaceName, vpnName, primaryRd, dpId);
 539             }
 540         }
 541     }
 542
 543     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 544         int lportTag, int addOrRemove) {
 545         Uuid extNetworkId;
 546         try {
 547             // vpn instance of ext-net interface is the network-id
 548             extNetworkId = new Uuid(vpnName);
 549         } catch (IllegalArgumentException e) {
 550             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 551                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 552             return;
 553         }
 554
 555         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 556         if (routerIds == null || routerIds.isEmpty()) {
 557             LOG.info("processExternalVpnInterface: No router is associated with {}."
 558                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 559                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 560             return;
 561         }
 562
 563         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 564                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 565         for (Uuid routerId : routerIds) {
 566             String routerName = routerId.getValue();
 567             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 568             if (Objects.equals(primarySwitch, dpId)) {
 569                 Routers router = vpnUtil.getExternalRouter(routerName);
 570                 if (router != null) {
 571                     if (addOrRemove == NwConstants.ADD_FLOW) {
 572                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 573                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 574                                 dpId, interfaceName, lportTag);
 575                     } else {
 576                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 577                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 578                                 dpId, interfaceName, lportTag);
 579                     }
 580                 } else {
 581                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 582                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 583                             interfaceName, dpId, vpnName);
 584                 }
 585             }
 586         }
 587     }
 588
 589     // TODO Clean up the exception handling
 590     @SuppressWarnings("checkstyle:IllegalCatch")
 591     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 592                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 593                                                   String vpnName, String interfaceName) {
 594         if (rd == null) {
 595             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 596                   interfaceName, dpnId, vpnName);
 597             return;
 598         } else {
 599             if (rd.equals(vpnName)) {
 600                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 601                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 602                 return;
 603             }
 604         }
 605         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 606                 interfaceName, dpnId, vpnName, rd);
 607
 608         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 609         if (nextHopIp == null) {
 610             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 611                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 612                     rd, vpnName);
 613             return;
 614         }
 615
 616         try {
 617             //Read NextHops
 618             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 619             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 620                     LogicalDatastoreType.OPERATIONAL, path);
 621             if (adjacencies.isPresent()) {
 622                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 623                 if (nextHops != null && !nextHops.isEmpty()) {
 624                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 625                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 626                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 627                     long l3vni = vpnInstanceOpData.getL3vni();
 628                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 629                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 630                     for (Adjacency nextHop : nextHops) {
 631                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 632                             continue;
 633                         }
 634                         String gatewayMac = null;
 635                         long label = 0;
 636                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 637                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 638                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 639                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 640                                     .get();
 641                         } else {
 642                             label = nextHop.getLabel();
 643                         }
 644                         try {
 645                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 646                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 647                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 648                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 649                                     gatewayMac);
 650                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 651                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 652                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 653                         } catch (Exception e) {
 654                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 655                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 656                                     interfaceName, dpnId, e);
 657                         }
 658                     }
 659                 }
 660             }
 661         } catch (ReadFailedException e) {
 662             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 663                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 664         }
 665     }
 666
 667     // TODO Clean up the exception handling
 668     @SuppressWarnings("checkstyle:IllegalCatch")
 669     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 670                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 671                         TypedWriteTransaction<Operational> writeOperTx) {
 672         //Read NextHops
 673         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 674         String rd = vpnUtil.getVpnRd(interfaceName);
 675         if (rd == null) {
 676             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 677                 interfaceName, vpnName);
 678             return;
 679         } else {
 680             if (rd.equals(vpnName)) {
 681                 LOG.info(
 682                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 683                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 684                 return;
 685             }
 686         }
 687         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 688                vpnName, rd);
 689         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 690         try {
 691             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 692                     path);
 693         } catch (ReadFailedException e) {
 694             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 695                     interfaceName, vpnName);
 696         }
 697         if (adjacencies.isPresent()) {
 698             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 699
 700             if (nextHops != null && !nextHops.isEmpty()) {
 701                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 702                         nextHops, interfaceName, vpnName, rd);
 703                 for (Adjacency nextHop : nextHops) {
 704                     try {
 705                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 706                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 707                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 708                                     interfaceName, vpnName);
 709                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 710                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 711                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 712                                     interfaceName, vpnName);
 713                         } else if (nextHop.getNextHopIpList() != null) {
 714                             // Perform similar operation as interface delete event for extraroutes.
 715                             String allocatedRd = nextHop.getVrfId();
 716                             for (String nh : nextHop.getNextHopIpList()) {
 717                                 deleteExtraRouteFromCurrentAndImportingVpns(
 718                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 719                                     writeOperTx);
 720                             }
 721                         }
 722                     } catch (Exception e) {
 723                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 724                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 725                     }
 726                 }
 727             }
 728         }
 729     }
 730
 731     @SuppressWarnings("checkstyle:IllegalCatch")
 732     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 733                                                   String primaryRd, String interfaceName, final long vpnId,
 734                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 735                                                   TypedWriteTransaction<Operational> writeOperTxn,
 736                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 737                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 738             throws ExecutionException, InterruptedException {
 739         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 740         // Read NextHops
 741         Optional<VpnInterface> vpnInteface = Optional.absent();
 742         try {
 743             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 744             LogicalDatastoreType.CONFIGURATION, identifier);
 745         } catch (ReadFailedException e) {
 746             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 747                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 748         }
 749         Uuid intfnetworkUuid = null;
 750         NetworkType networkType = null;
 751         Long segmentationId = Long.valueOf(-1);
 752         Adjacencies adjacencies = null;
 753         if (vpnInteface.isPresent()) {
 754             intfnetworkUuid = vpnInteface.get().getNetworkId();
 755             networkType = vpnInteface.get().getNetworkType();
 756             segmentationId = vpnInteface.get().getSegmentationId();
 757             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 758             if (adjacencies == null) {
 759                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 760                         null/*gwMac*/, writeOperTxn);
 761                 return;
 762             }
 763         }
 764         // Get the rd of the vpn instance
 765         String nextHopIp = null;
 766         try {
 767             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 768         } catch (Exception e) {
 769             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 770                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 771         }
 772         List<String> nhList = new ArrayList<>();
 773         if (nextHopIp != null) {
 774             nhList.add(nextHopIp);
 775             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 776                     interfaceName, dpnId, vpnName, nhList);
 777         }
 778         Optional<String> gwMac = Optional.absent();
 779         String vpnInterfaceSubnetGwMacAddress = null;
 780         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 781         Long l3vni = vpnInstanceOpData.getL3vni();
 782         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 783         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 784         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 785         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 786         List<Adjacency> value = new ArrayList<>();
 787         for (Adjacency nextHop : nextHops) {
 788             String rd = primaryRd;
 789             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 790             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 791                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 792                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 793                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 794                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 795                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 796                 continue;
 797             }
 798             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 799                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 800                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 801                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 802                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 803                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 804
 805                 Prefixes prefixes = intfnetworkUuid != null
 806                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 807                             segmentationId, prefixCue) :
 808                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 809                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 810                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 811                 final Uuid subnetId = nextHop.getSubnetId();
 812
 813                 String gatewayIp = nextHop.getSubnetGatewayIp();
 814                 if (gatewayIp == null) {
 815                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 816                     if (gatewayIpOptional.isPresent()) {
 817                         gatewayIp = gatewayIpOptional.get();
 818                     }
 819                 }
 820
 821                 if (gatewayIp != null) {
 822                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 823                     if (gwMac.isPresent()) {
 824                         // A valid mac-address is available for this subnet-gateway-ip
 825                         // Use this for programming ARP_RESPONDER table here.  And save this
 826                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 827                         // to populate L3_GW_MAC_TABLE there.
 828                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 829                                 gatewayIp, gwMac.get());
 830                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 831                     } else {
 832                         // A valid mac-address is not available for this subnet-gateway-ip
 833                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 834                         // Save this connected-mac-address as gateway-mac-address for the
 835                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 836                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 837                         if (gwMac.isPresent()) {
 838                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 839                                     NwConstants.ADD_FLOW, gwMac.get());
 840                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 841                                     gatewayIp, gwMac.get());
 842                         } else {
 843                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 844                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 845                                 + "gwIp {}",
 846                                 subnetId, interfaceName, vpnName, gatewayIp);
 847                         }
 848                     }
 849                 } else {
 850                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 851                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 852                         subnetId, interfaceName, vpnName);
 853                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 854                 }
 855                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 856                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 857             } else {
 858                 //Extra route adjacency
 859                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 860                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 861                 // FIXME: separate this out somehow?
 862                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 863                 lock.lock();
 864                 try {
 865                     java.util.Optional<String> rdToAllocate = vpnUtil
 866                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 867                                     nextHop.getNextHopIpList().get(0), dpnId);
 868                     if (rdToAllocate.isPresent()) {
 869                         rd = rdToAllocate.get();
 870                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 871                             rd, prefix);
 872                     } else {
 873                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 874                         continue;
 875                     }
 876                 } finally {
 877                     lock.unlock();
 878                 }
 879                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 880                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 881                         interfaceName, dpnId);
 882             }
 883             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 884             // can be different from the gateway-mac-address within the VRFEntry as the
 885             // gateway-mac-address is a superset.
 886             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 887             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 888                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 889                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 890             Adjacency operationalAdjacency = null;
 891             try {
 892                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 893             } catch (NullPointerException e) {
 894                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 895                     input, e.getMessage());
 896                 return;
 897             }
 898             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 899                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 900                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 901                     writeConfigTxn);
 902             }
 903             value.add(operationalAdjacency);
 904         }
 905
 906         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 907         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 908                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 909
 910         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 911                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 912                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 913
 914         for (Adjacency nextHop : aug.getAdjacency()) {
 915             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 916             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 917                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 918                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 919                 registeredPopulator.populateFib(input, writeConfigTxn);
 920             }
 921         }
 922     }
 923
 924     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 925                                               long lportTag, String gwMac,
 926                                               TypedWriteTransaction<Operational> writeOperTxn) {
 927         VpnInterfaceOpDataEntry opInterface =
 928               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 929         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 930             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 931         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 932         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 933                 interfaceName, dpnId, vpnName);
 934     }
 935
 936     // TODO Clean up the exception handling
 937     @SuppressWarnings("checkstyle:IllegalCatch")
 938     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 939                                            StateTunnelList stateTunnelList,
 940                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 941                                            TypedWriteTransaction<Operational> writeOperTxn) {
 942
 943         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 944         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 945         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 946         List<Adjacency> adjList =
 947             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 948         if (adjList.isEmpty()) {
 949             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 950                     vpnInterface, srcDpnId);
 951             return;
 952         }
 953         String prefix = null;
 954         long label = 0;
 955         List<Adjacency> value = new ArrayList<>();
 956         boolean isNextHopAddReqd = false;
 957         String vpnName = vpnInterface.getVpnInstanceName();
 958         long vpnId = vpnUtil.getVpnId(vpnName);
 959         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 960         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 961                 vpnInterface.getName(), vpnInterface.getDpnId(),
 962                 vpnInterface.getVpnInstanceName(), adjList);
 963         for (Adjacency adj : adjList) {
 964             String rd = adj.getVrfId();
 965             rd = rd != null ? rd : vpnName;
 966             prefix = adj.getIpAddress();
 967             label = adj.getLabel();
 968             List<String> nhList = Collections.singletonList(srcTepIp);
 969             List<String> nextHopList = adj.getNextHopIpList();
 970             // If TEP is added , update the nexthop of primary adjacency.
 971             // Secondary adj nexthop is already pointing to primary adj IP address.
 972             if (nextHopList == null || nextHopList.isEmpty()) {
 973                 isNextHopAddReqd = true;
 974             }
 975
 976             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 977                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 978             } else {
 979                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 980                 if (!vrfEntryOptional.isPresent()) {
 981                     continue;
 982                 }
 983                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 984                 if (!nhList.contains(srcTepIp)) {
 985                     nhList.add(srcTepIp);
 986                     isNextHopAddReqd = true;
 987                 }
 988                 value.add(adj);
 989             }
 990
 991             if (isNextHopAddReqd) {
 992                 updateLabelMapper(label, nhList);
 993                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 994                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 995                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 996                 // Update the VRF entry with nextHop
 997                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 998                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 999
1000                 //Get the list of VPN's importing this route(prefix) .
1001                 // Then update the VRF entry with nhList
1002                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1003                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1004                     String vpnRd = vpn.getVrfId();
1005                     if (vpnRd != null) {
1006                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1007                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1008                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1009                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1010                             vpnInterface.getName(), srcDpnId, vpnName,
1011                             vpn.getVpnInstanceName(), vpnRd);
1012                     }
1013                 }
1014                 // Advertise the prefix to BGP only for external vpn
1015                 // since there is a nexthop change.
1016                 try {
1017                     if (!rd.equalsIgnoreCase(vpnName)) {
1018                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1019                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1020                                 null /*gatewayMacAddress*/);
1021                     }
1022                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1023                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1024                             srcDpnId, vpnName);
1025                 } catch (Exception ex) {
1026                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1027                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1028                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1029                 }
1030             }
1031         }
1032         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1033         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1034                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1035                 .addAugmentation(AdjacenciesOp.class, aug).build();
1036         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1037                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1038         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1039         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1040                 vpnInterface.getName(), srcDpnId, vpnName);
1041
1042     }
1043
1044     // TODO Clean up the exception handling
1045     @SuppressWarnings("checkstyle:IllegalCatch")
1046     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1047                                               StateTunnelList stateTunnelList,
1048                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1049                                               TypedWriteTransaction<Operational> writeOperTxn) {
1050
1051         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1052         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1053         String prefix = null;
1054         long label = 0;
1055         boolean isNextHopRemoveReqd = false;
1056         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1057         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1058         String vpnName = vpnInterface.getVpnInstanceName();
1059         long vpnId = vpnUtil.getVpnId(vpnName);
1060         String primaryRd = vpnUtil.getVpnRd(vpnName);
1061         if (adjList != null) {
1062             List<Adjacency> value = new ArrayList<>();
1063             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1064                     vpnInterface.getName(), vpnInterface.getDpnId(),
1065                     vpnInterface.getVpnInstanceName(), adjList);
1066             for (Adjacency adj : adjList) {
1067                 List<String> nhList = new ArrayList<>();
1068                 String rd = adj.getVrfId();
1069                 rd = rd != null ? rd : vpnName;
1070                 prefix = adj.getIpAddress();
1071                 List<String> nextHopList = adj.getNextHopIpList();
1072                 label = adj.getLabel();
1073                 if (nextHopList != null && !nextHopList.isEmpty()) {
1074                     isNextHopRemoveReqd = true;
1075                 }
1076                 // If TEP is deleted , remove the nexthop from primary adjacency.
1077                 // Secondary adj nexthop will continue to point to primary adj IP address.
1078                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1079                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1080                 } else {
1081                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1082                     if (!vrfEntryOptional.isPresent()) {
1083                         continue;
1084                     }
1085                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1086                     if (nhList.contains(srcTepIp)) {
1087                         nhList.remove(srcTepIp);
1088                         isNextHopRemoveReqd = true;
1089                     }
1090                     value.add(adj);
1091                 }
1092
1093                 if (isNextHopRemoveReqd) {
1094                     updateLabelMapper(label, nhList);
1095                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1096                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1097                             prefix, nhList, vpnName,
1098                             vpnId, rd, vpnInterface.getName());
1099                     // Update the VRF entry with removed nextHop
1100                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1101                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1102
1103                     //Get the list of VPN's importing this route(prefix) .
1104                     // Then update the VRF entry with nhList
1105                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1106                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1107                         String vpnRd = vpn.getVrfId();
1108                         if (vpnRd != null) {
1109                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1110                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1111                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1112                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1113                                     nhList, label, vpnInterface.getName(), srcDpnId,
1114                                     vpnName,
1115                                     vpn.getVpnInstanceName(), vpnRd);
1116                         }
1117                     }
1118
1119                     // Withdraw prefix from BGP only for external vpn.
1120                     try {
1121                         if (!rd.equalsIgnoreCase(vpnName)) {
1122                             bgpManager.withdrawPrefix(rd, prefix);
1123                         }
1124                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1125                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1126                                 vpnInterface.getName(), srcDpnId,
1127                                 vpnName);
1128                     } catch (Exception ex) {
1129                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1130                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1131                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1132                     }
1133                 }
1134             }
1135             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1136             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1137                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1138                     .addAugmentation(AdjacenciesOp.class, aug).build();
1139             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1140                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1141             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1142             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1143                          vpnInterface.getName(), srcDpnId, vpnName);
1144         }
1145     }
1146
1147     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1148         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1149
1150         String vpnRd = vpnUtil.getVpnRd(vpnName);
1151         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1152         if (vpnInstanceOpDataEntry == null) {
1153             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1154                     + " to check for vpns exporting the routes", vpnName);
1155             return vpnsToExportRoute;
1156         }
1157
1158         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1159             if (input.getVpnInstanceName() == null) {
1160                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1161                         input.getVrfId());
1162                 return false;
1163             }
1164             return !input.getVpnInstanceName().equals(vpnName);
1165         };
1166
1167         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1168             Iterable<String> commonRTs =
1169                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1170                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1171             return Iterators.size(commonRTs.iterator()) > 0;
1172         };
1173
1174         vpnsToExportRoute =
1175                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1176                         Collectors.toList());
1177         return vpnsToExportRoute;
1178     }
1179
1180     // TODO Clean up the exception handling
1181     @SuppressWarnings("checkstyle:IllegalCatch")
1182     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1183         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1184         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1185             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1186             if (vrfEntries != null) {
1187                 ListenableFutures.addErrorLogging(
1188                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1189                         for (VrfEntry vrfEntry : vrfEntries) {
1190                             try {
1191                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1192                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1193                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1194                                             + " is not a controller managed non intervpn link route. Ignoring.",
1195                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1196                                     continue;
1197                                 }
1198                                 String prefix = vrfEntry.getDestPrefix();
1199                                 String gwMac = vrfEntry.getGatewayMacAddress();
1200                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1201                                     String nh = routePath.getNexthopAddress();
1202                                     int label = routePath.getLabel().intValue();
1203                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1204                                         vrfEntry.getOrigin()))) {
1205                                         LOG.info(
1206                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1207                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1208                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1209                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1210                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1211                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1212                                             confTx);
1213                                     } else {
1214                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1215                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1216                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1217                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1218                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1219                                             confTx);
1220                                     }
1221                                 });
1222                             } catch (RuntimeException e) {
1223                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1224                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1225                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1226                             }
1227                         }
1228                     }), LOG, "Error handing VPN exporting routes");
1229             } else {
1230                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1231                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1232             }
1233         }
1234     }
1235
1236     @Override
1237     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1238         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1239         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1240         final String interfaceName = key.getName();
1241         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1242             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1243             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1244         }
1245     }
1246
1247     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1248                                 final VpnInterface vpnInterface, final String vpnName,
1249                                 final String interfaceName) {
1250         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1251             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1252                 ListenableFuture<Void> future =
1253                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1254                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1255                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1256                     });
1257                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1258                         vpnInterface.getName(), vpnName);
1259                 return Collections.singletonList(future);
1260             }, DJC_MAX_RETRIES);
1261         } else {
1262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1263             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1264         }
1265     }
1266
1267     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1268     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1269                                            final VpnInterface vpnInterface, final String vpnName,
1270                                            final String interfaceName, final Interface interfaceState) {
1271         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1272                 vpnName, vpnInterface.getDpnId());
1273         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1274         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1275             () -> {
1276                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1277                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1278                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1279                         writeOperTxn -> futures.add(
1280                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1281                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1282                                         interfaceName, vpnName);
1283                                 BigInteger dpId;
1284                                 int ifIndex;
1285                                 String gwMacAddress;
1286                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1287                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1288                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1289                                 try {
1290                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1291                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1292                                 } catch (ReadFailedException e) {
1293                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1294                                             interfaceName, vpnName);
1295                                     return;
1296                                 }
1297                                 if (interfaceState != null) {
1298                                     try {
1299                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1300                                     } catch (NumberFormatException | IllegalStateException e) {
1301                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1302                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1303                                                         + " from vpn interface op data store. ", interfaceName,
1304                                                 vpnInterface.getDpnId(), vpnName, e);
1305                                         dpId = BigInteger.ZERO;
1306                                     }
1307                                     ifIndex = interfaceState.getIfIndex();
1308                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1309                                 } else {
1310                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1311                                             + " from vpn interface op.", interfaceName);
1312                                     if (optVpnInterface.isPresent()) {
1313                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1314                                         dpId = vpnOpInterface.getDpnId();
1315                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1316                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1317                                     } else {
1318                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1319                                                 + " as interfaceState and vpn interface op is not"
1320                                                 + " available", interfaceName, vpnName);
1321                                         return;
1322                                     }
1323                                 }
1324                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1325                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1326                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1327                                 LOG.info(
1328                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1329                                                 + "successfully",
1330                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1331                             })))));
1332                 futures.add(configFuture);
1333                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1334                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1335                 return futures;
1336             }, DJC_MAX_RETRIES);
1337     }
1338
1339     protected void processVpnInterfaceDown(BigInteger dpId,
1340                                            String interfaceName,
1341                                            int lportTag,
1342                                            String gwMac,
1343                                            VpnInterfaceOpDataEntry vpnOpInterface,
1344                                            boolean isInterfaceStateDown,
1345                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1346                                            TypedWriteTransaction<Operational> writeOperTxn,
1347                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1348             throws ExecutionException, InterruptedException {
1349         if (vpnOpInterface == null) {
1350             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1351                     + " as it is not available in operational data store", interfaceName, dpId);
1352             return;
1353         }
1354         final String vpnName = vpnOpInterface.getVpnInstanceName();
1355         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1356                                                     interfaceName, vpnName);
1357         if (!isInterfaceStateDown) {
1358             final long vpnId = vpnUtil.getVpnId(vpnName);
1359             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1360             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1361             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1362                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1363             if (interfaceManager.isExternalInterface(interfaceName)) {
1364                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1365                     NwConstants.DEL_FLOW);
1366             }
1367             if (!isBgpVpnInternetVpn) {
1368                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1369             }
1370             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1371                     + " successful", interfaceName, dpId, vpnName);
1372         } else {
1373             // Interface is retained in the DPN, but its Link Down.
1374             // Only withdraw the prefixes for this interface from BGP
1375             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1376         }
1377     }
1378
1379     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1380                                           final String vpnName, final long vpnId, String gwMac,
1381                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1382                                           TypedWriteTransaction<Operational> writeOperTxn,
1383                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1384             throws ExecutionException, InterruptedException {
1385         //Read NextHops
1386         try {
1387             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1388                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1389             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1390             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1391                     LogicalDatastoreType.OPERATIONAL, path);
1392             boolean isNonPrimaryAdjIp = Boolean.FALSE;
1393             String primaryRd = vpnUtil.getVpnRd(vpnName);
1394             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1395                     interfaceName, dpnId, vpnName, primaryRd);
1396             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1397                     && !adjacencies.get().getAdjacency().isEmpty()) {
1398                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1399                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1400                         interfaceName, dpnId, vpnName, nextHops);
1401                 for (Adjacency nextHop : nextHops) {
1402                     if (nextHop.isPhysNetworkFunc()) {
1403                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1404                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1405                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1406                                 null/*writeCfgTxn*/);
1407                     } else {
1408                         String rd = nextHop.getVrfId();
1409                         List<String> nhList;
1410                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1411                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1412                             isNonPrimaryAdjIp = Boolean.TRUE;
1413                         } else {
1414                             // This is a primary adjacency
1415                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1416                                     : emptyList();
1417                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1418                                     interfaceName, writeInvTxn);
1419                         }
1420                         if (!nhList.isEmpty()) {
1421                             if (Objects.equals(primaryRd, vpnName)) {
1422                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1423                                 //remove from FIB directly
1424                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1425                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1426                             } else {
1427                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1428                                         interfaceName, writeConfigTxn, writeOperTxn);
1429                             }
1430                         } else {
1431                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1432                                             + " interface {}", nextHop.getIpAddress(), rd,
1433                                     nextHop.getAdjacencyType().toString(), interfaceName);
1434                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1435                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1436                         }
1437                     }
1438                     String ip = nextHop.getIpAddress().split("/")[0];
1439                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1440                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1441                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1442                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1443                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1444                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1445                     }
1446                     // Remove the MIP-IP from VpnPortIpToPort.
1447                     if (isNonPrimaryAdjIp) {
1448                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1449                         if (persistedIp != null && persistedIp.isLearntIp()
1450                                 && persistedIp.getPortName().equals(interfaceName)) {
1451                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1452                             LOG.info(
1453                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1454                                             + "from VpnPortipToPort",
1455                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1456                         }
1457                     }
1458                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1459                     if (vpnPortipToPort != null) {
1460                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1461                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1462                                  + "Interface {} ip {} on dpn {} for vpn {}",
1463                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1464                     }
1465                 }
1466             } else {
1467                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1468                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1469                         + " Removing it.", interfaceName, vpnName, dpnId);
1470                 writeOperTxn.delete(identifier);
1471             }
1472         } catch (ReadFailedException e) {
1473             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1474                     interfaceName, dpnId, vpnName);
1475         }
1476     }
1477
1478     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1479                                                             String interfaceName, BigInteger dpnId,
1480                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1481                                                             TypedWriteTransaction<Operational> writeOperTx) {
1482         return (nh) -> {
1483             String primaryRd = vpnUtil.getVpnRd(vpnName);
1484             String prefix = nextHop.getIpAddress();
1485             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1486             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1487                     nextHop, vpnName, interfaceName, dpnId);
1488             // FIXME: separate this out somehow?
1489             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1490             lock.lock();
1491             try {
1492                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1493                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1494                     //If extra-route is present behind at least one VM, then do not remove or update
1495                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1496                     //prefixtointerface DS
1497                     return;
1498                 }
1499                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1500                         writeConfigTxn);
1501             } finally {
1502                 lock.unlock();
1503             }
1504             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1505                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1506                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1507         };
1508     }
1509
1510     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1511                                            BigInteger dpnId, String rd, String interfaceName,
1512                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1513                                            TypedWriteTransaction<Operational> writeOperTx) {
1514         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1515                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1516         nhList.forEach((nh) -> {
1517             //IRT: remove routes from other vpns importing it
1518             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1519                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1520             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1521                 String vpnRd = vpn.getVrfId();
1522                 if (vpnRd != null) {
1523                     fibManager.removeOrUpdateFibEntry(vpnRd,
1524                             nextHop.getIpAddress(), nh, writeConfigTxn);
1525                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1526                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1527                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1528                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1529                 }
1530             }
1531         });
1532     }
1533
1534     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1535                                                  int lportTag, String gwMac, String interfaceName,
1536                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1537             throws ExecutionException, InterruptedException {
1538         final Uuid subnetId = nextHop.getSubnetId();
1539         if (nextHop.getSubnetGatewayMacAddress() == null) {
1540             // A valid mac-address was not available for this subnet-gateway-ip
1541             // So a connected-mac-address was used for this subnet and we need
1542             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1543             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1544         }
1545         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1546                 subnetId);
1547     }
1548
1549     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1550                                                   String interfaceName) {
1551         // This is either an extra-route (or) a learned IP via subnet-route
1552         List<String> nhList = null;
1553         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1554         if (nextHopIp == null || nextHopIp.isEmpty()) {
1555             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1556                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1557                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1558                     vpnName);
1559             nhList = emptyList();
1560         } else {
1561             nhList = Collections.singletonList(nextHopIp);
1562         }
1563         return nhList;
1564     }
1565
1566     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1567         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1568         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1569         // else use connected interface
1570         if (gwPort != null && gwPort.isSubnetIp()) {
1571             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1572                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1573             return Optional.of(gwPort.getMacAddress());
1574         }
1575         return Optional.absent();
1576     }
1577
1578     @Override
1579     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1580         final VpnInterface update) {
1581         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1582         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1583                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1584         if (original.equals(update)) {
1585             LOG.info("update: original {} update {} are same. No update required.", original, update);
1586             return;
1587         }
1588         final String vpnInterfaceName = update.getName();
1589         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1590         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1591         //handles switching between <internal VPN - external VPN>
1592         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1593             List<ListenableFuture<Void>> futures = new ArrayList<>();
1594             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1595                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1596                                 + "to newVpn(s) {}",
1597                         original.getName(), dpnId,
1598                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1599                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1600                 return emptyList();
1601             }
1602             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1603             return futures;
1604         });
1605     }
1606
1607     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1608                                                            VpnInterface original, VpnInterface update,
1609                                                            List<ListenableFuture<Void>> futures) {
1610         boolean isVpnInstanceUpdate = false;
1611         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1612         final String interfaceName = key.getName();
1613         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1614         List<String> oldVpnListCopy = new ArrayList<>();
1615         oldVpnListCopy.addAll(oldVpnList);
1616         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1617         List<String> newVpnListCopy = new ArrayList<>();
1618         newVpnListCopy.addAll(newVpnList);
1619
1620         oldVpnList.removeAll(newVpnList);
1621         newVpnList.removeAll(oldVpnListCopy);
1622         //This block will execute only on if there is a change in the VPN Instance.
1623         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1624             /*
1625              * Internet BGP-VPN Instance update with single router:
1626              * ====================================================
1627              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1628              *     1st VPN Instance : router VPN or external BGP-VPN.
1629              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1630              *
1631              * VPN Instance UPDATE:
1632              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1633              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1634              *
1635              * External BGP-VPN Instance update with single router:
1636              * ====================================================
1637              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1638              *
1639              * Updated VPN Instance will be always either internal router VPN to
1640              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1641              *
1642              * VPN Instance UPDATE:
1643              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1644              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1645              *
1646              * Dual Router VPN Instance Update:
1647              * ================================
1648              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1649              *
1650              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1651              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1652              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1653              *
1654              * Dual Router --> Associated with common external BGP-VPN Instance.
1655              * 1st router and 2nd router are getting associated with single External BGP-VPN
1656              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1657              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1658              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1659              *
1660              *
1661              */
1662             isVpnInstanceUpdate = true;
1663             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1664                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1665                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1666                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1667                     List<String> externalBgpVpnList = new ArrayList<>();
1668                     for (String newVpnName : newVpnListCopy) {
1669                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1670                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1671                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1672                                 .BgpvpnType.BGPVPNExternal) {
1673                             externalBgpVpnList.add(newVpnName);
1674                             break;
1675                         }
1676                     }
1677                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1678                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1679                             externalBgpVpnList, oldVpnListCopy, futures);
1680
1681                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1682                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1683                     //Identify the router VPN Instance and pass that value as oldVpnList
1684                     List<String> routerVpnList = new ArrayList<>();
1685                     for (String newVpnName : newVpnListCopy) {
1686                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1687                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1688                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1689                                 .BgpvpnType.VPN) {
1690                             routerVpnList.add(newVpnName);
1691                             break;
1692                         }
1693                     }
1694                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1695                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1696                             newVpnList, oldVpnListCopy, futures);
1697
1698                 } else {
1699                     //Handle remaining use cases.
1700                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1701                             oldVpnListCopy, futures);
1702                 }
1703             } else {
1704                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1705                         oldVpnListCopy, futures);
1706             }
1707         }
1708         return isVpnInstanceUpdate;
1709     }
1710
1711     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1712                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1713                                          List<String> newVpnList, List<String> oldVpnListCopy,
1714                                          List<ListenableFuture<Void>> futures) {
1715         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1716         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1717                 ? origAdjs.getAdjacency() : new ArrayList<>();
1718         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1719         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1720                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1721
1722         boolean isOldVpnRemoveCallExecuted = false;
1723         for (String oldVpnName : oldVpnList) {
1724             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1725                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1726             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1727             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1728                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1729                     newVpnList);
1730             isOldVpnRemoveCallExecuted = true;
1731         }
1732         //Wait for previous interface bindings to be removed
1733         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1734             try {
1735                 Thread.sleep(2000);
1736             } catch (InterruptedException e) {
1737                 //Ignore
1738             }
1739         }
1740         for (String newVpnName : newVpnList) {
1741             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1742             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1743                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1744                         + "onto vpnName {} ", interfaceName, newVpnName);
1745                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1746                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1747                                 + "from oldVpn(s) {} to newVpn {} ",
1748                         interfaceName, oldVpnListCopy, newVpnName);
1749                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1750                  * Use Case:
1751                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1752                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1753                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1754                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1755                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1756                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1757                  *     router VPN as well as Internet BGP-VPN.
1758                  *
1759                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1760                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1761                  *                                       update in existing router VPN instance.
1762                  */
1763                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1764                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1765                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1766                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1767                 }
1768             }
1769         }
1770     }
1771
1772     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1773                                                                     String vpnInterfaceName,
1774                                                                     List<ListenableFuture<Void>> futures) {
1775         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1776         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1777                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1778         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1779         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1780                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1781
1782         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1783         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1784             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1785             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1786             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1787             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1788             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1789                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1790                 //set of prefix used as entry in prefix-to-interface datastore
1791                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1792                 // in bucket actions on bgp-vpn delete
1793                 Set<String> prefixListForRefreshFib = new HashSet<>();
1794                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1795                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1796                         operTx -> {
1797                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1798                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1799                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1800                                     update.getName(), newVpnName);
1801                             //handle both addition and removal of adjacencies
1802                             // currently, new adjacency may be an extra route
1803                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1804                             if (!oldAdjs.equals(newAdjs)) {
1805                                 for (Adjacency adj : copyNewAdjs) {
1806                                     if (copyOldAdjs.contains(adj)) {
1807                                         copyOldAdjs.remove(adj);
1808                                     } else {
1809                                         // add new adjacency
1810                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1811                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1812                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1813                                         }
1814                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1815                                             + " added to vpn interface {} on vpn {} dpnId {}",
1816                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1817                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1818                                     }
1819                                 }
1820                                 for (Adjacency adj : copyOldAdjs) {
1821                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1822                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1823                                             && !adj.isPhysNetworkFunc()) {
1824                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1825                                                 confTx);
1826                                             //remove FIB entry
1827                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1828                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1829                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1830                                             //remove BGP entry
1831                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1832                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1833                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1834                                             }
1835                                         } else {
1836                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1837                                                 operTx, confTx);
1838                                         }
1839                                     }
1840                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1841                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1842                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1843                                 }
1844                             }
1845                         })));
1846                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1847                     MoreExecutors.directExecutor());
1848                 futures.add(configTxFuture);
1849                 for (ListenableFuture<Void> future : futures) {
1850                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1851                             update.getName(), update.getVpnInstanceNames());
1852                 }
1853             } else {
1854                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1855                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1856             }
1857         }
1858         return futures;
1859     }
1860
1861     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1862         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1863                 .toString();
1864         // FIXME: separate this out somehow?
1865         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1866         lock.lock();
1867         try {
1868             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1869                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1870             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1871                 LogicalDatastoreType.OPERATIONAL, lriIid);
1872             if (opResult.isPresent()) {
1873                 LabelRouteInfo labelRouteInfo =
1874                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1875                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1876                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1877             }
1878             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1879                     nextHopIpList);
1880         } catch (ReadFailedException e) {
1881             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1882                     nextHopIpList);
1883         } catch (TransactionCommitFailedException e) {
1884             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1885                     nextHopIpList);
1886         } finally {
1887             lock.unlock();
1888         }
1889     }
1890
1891     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1892         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1893
1894         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1895         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1896                 .addAugmentation(SubnetRoute.class, route).build();
1897         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1898         InstanceIdentifierBuilder<VrfTables> idBuilder =
1899             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1900         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1901         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1902         if (writeConfigTxn != null) {
1903             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1904         } else {
1905             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1906         }
1907         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1908                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1909     }
1910
1911     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1912                                            Adjacency adj, BigInteger dpnId,
1913                                            TypedWriteTransaction<Operational> writeOperTxn,
1914                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1915                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1916                                            Set<String> prefixListForRefreshFib)
1917             throws ExecutionException, InterruptedException {
1918         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1919         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1920         try {
1921             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1922                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1923             if (optVpnInterface.isPresent()) {
1924                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1925                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1926                 String vpnName = currVpnIntf.getVpnInstanceName();
1927                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1928                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1929                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1930                         LogicalDatastoreType.OPERATIONAL, adjPath);
1931                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1932                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1933                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1934                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1935                 List<Adjacency> adjacencies = new ArrayList<>();
1936                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1937                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1938                 }
1939                 long vpnId = vpnUtil.getVpnId(vpnName);
1940                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1941                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1942                 Adjacency operationalAdjacency = null;
1943                 //Handling dual stack neutron port primary adjacency
1944                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1945                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1946                             currVpnIntf.getName(), vpnName);
1947                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1948                             currVpnIntf.getName());
1949                     if (interfaceState != null) {
1950                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1951                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1952                             prefixListForRefreshFib);
1953                     }
1954                 }
1955                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1956                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1957                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1958                             : RouteOrigin.STATIC;
1959                     String nh = adj.getNextHopIpList().get(0);
1960                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1961                     // FIXME: separate out to somehow?
1962                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1963                     lock.lock();
1964                     try {
1965                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1966                                 vpnId, null, prefix, vpnName, nh, dpnId);
1967                         if (rdToAllocate.isPresent()) {
1968                             input.setRd(rdToAllocate.get());
1969                             operationalAdjacency = populator.createOperationalAdjacency(input);
1970                             int label = operationalAdjacency.getLabel().intValue();
1971                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1972                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1973                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1974                                     prefixListForRefreshFib, writeConfigTxn);
1975                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1976                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1977                                     vpnName, label, currVpnIntf.getName(), dpnId);
1978                         } else {
1979                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1980                                     vpnName, prefix);
1981                             return;
1982                         }
1983                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1984                         // Keeping the MPLS check for now.
1985                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1986                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1987                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1988                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1989                             vpnsToImportRoute.forEach(vpn -> {
1990                                 if (vpn.getVrfId() != null) {
1991                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1992                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1993                                             .ifPresent(
1994                                                 rds -> vpnManager.addExtraRoute(
1995                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1996                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1997                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1998                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1999                                 }
2000                             });
2001                         }
2002                     } finally {
2003                         lock.unlock();
2004                     }
2005                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2006                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2007                             currVpnIntf.getName(), vpnName);
2008
2009                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2010                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2011                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2012                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2013                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
2014                             Prefixes.PrefixCue.PhysNetFunc);
2015                     if (vpnIntefaceConfig.isPresent()) {
2016                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
2017                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2018                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
2019                     }
2020
2021                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2022
2023                     writeOperTxn.merge(
2024                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2025                                     prefix), pnfPrefix, true);
2026
2027                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2028                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
2029                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
2030
2031                     input.setRd(adj.getVrfId());
2032                 }
2033                 if (operationalAdjacency == null) {
2034                     operationalAdjacency = populator.createOperationalAdjacency(input);
2035                 }
2036                 adjacencies.add(operationalAdjacency);
2037                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2038                 VpnInterfaceOpDataEntry newVpnIntf =
2039                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2040                                 aug, dpnId, currVpnIntf.getLportTag(),
2041                                 currVpnIntf.getGatewayMacAddress());
2042
2043                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2044             }
2045         } catch (ReadFailedException e) {
2046             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2047                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2048         }
2049     }
2050
2051     @Nullable
2052     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2053         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2054         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2055     }
2056
2057     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2058                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2059                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2060         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2061         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2062         try {
2063             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2064                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2065             if (optVpnInterface.isPresent()) {
2066                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2067                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2068                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2069                         LogicalDatastoreType.OPERATIONAL, path);
2070                 if (optAdjacencies.isPresent()) {
2071                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2072
2073                     if (adjacencies != null && !adjacencies.isEmpty()) {
2074                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2075                         for (Adjacency adjacency : adjacencies) {
2076                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2077                                 String rd = adjacency.getVrfId();
2078                                 InstanceIdentifier<Adjacency> adjIdentifier = VpnUtil
2079                                         .getVpnInterfaceOpDataEntryAdjacencyIdentifier(currVpnIntf.getName(),
2080                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress());
2081                                 LOG.debug("delAdjFromVpnInterface: adjIdentifier {}", adjIdentifier);
2082                                 writeOperTxn.delete(adjIdentifier);
2083                                 if (adj.getNextHopIpList() != null) {
2084                                     for (String nh : adj.getNextHopIpList()) {
2085                                         deleteExtraRouteFromCurrentAndImportingVpns(
2086                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2087                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2088                                     }
2089                                 } else if (adj.isPhysNetworkFunc()) {
2090                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2091                                             adj.getIpAddress(), adj.getSubnetId());
2092                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2093                                             writeConfigTxn);
2094                                 }
2095                                 break;
2096                             }
2097
2098                         }
2099                     }
2100                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2101                             dpnId, adj.getVrfId());
2102                 } else {
2103                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2104                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2105                 }
2106             }
2107         } catch (ReadFailedException e) {
2108             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2109                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2110         }
2111     }
2112
2113     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2114                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2115                                     TypedWriteTransaction<Operational> writeOperTx) {
2116         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2117         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2118         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2119             String vpnRd = vpn.getVrfId();
2120             if (vpnRd != null) {
2121                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2122                         writeOperTx);
2123             }
2124         }
2125     }
2126
2127     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2128         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2129             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2130             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2131     }
2132
2133     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2134         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2135             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2136     }
2137
2138     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2139                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2140         if (vpnInterface == null) {
2141             return;
2142         }
2143         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2144         if (adjs == null) {
2145             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2146                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2147             return;
2148         }
2149         for (Adjacency adj : adjs) {
2150             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2151                 String primaryInterfaceIp = adj.getIpAddress();
2152                 String macAddress = adj.getMacAddress();
2153                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2154
2155                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2156                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2157
2158                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2159                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2160                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2161                         routerInt, label, writeConfigTxn);
2162                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2163                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2164                         macAddress);
2165             } else {
2166                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2167                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2168                         interfaceName, primaryRd, vpnName);
2169             }
2170         }
2171     }
2172
2173     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2174             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2175         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2176         String rd = vpnUtil.getVpnRd(vpnName);
2177         if (adjs != null) {
2178             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2179             for (Adjacency adj : adjsList) {
2180                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2181                     String primaryInterfaceIp = adj.getIpAddress();
2182                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2183                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2184                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2185                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2186                 }
2187             }
2188         } else {
2189             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2190                     vpnInterface.getName(), rd);
2191         }
2192     }
2193
2194     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2195         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2196         final String interfaceName = key.getName();
2197         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2198                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2199         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2200                   intefaceData.identifier, vpnName);
2201     }
2202
2203     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2204                                               VpnInterface vpnInterface, String vpnName) {
2205         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2206                .get(vpnName);
2207         if (vpnInterfaces == null) {
2208             vpnInterfaces = new ConcurrentLinkedQueue<>();
2209         }
2210         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2211         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2212         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2213                 vpnInterface.getName(), vpnName);
2214     }
2215
2216     public boolean isVpnInstanceReady(String vpnInstanceName) {
2217         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2218         if (vpnRd == null) {
2219             return false;
2220         }
2221         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2222
2223         return vpnInstanceOpDataEntry != null;
2224     }
2225
2226     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2227         // FIXME: separate out to somehow?
2228         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2229         lock.lock();
2230         try {
2231             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2232                     unprocessedVpnInterfaces.get(vpnInstanceName);
2233             if (vpnInterfaces != null) {
2234                 while (!vpnInterfaces.isEmpty()) {
2235                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2236                     if (hasVpnInstanceCreatedSuccessfully) {
2237                         processSavedInterface(savedInterface, vpnInstanceName);
2238                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2239                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2240                     } else {
2241                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2242                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2243                     }
2244                 }
2245             } else {
2246                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2247             }
2248         } finally {
2249             lock.unlock();
2250         }
2251     }
2252
2253     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2254             VpnInterface vpnInterface) {
2255         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2256         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2257         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2258         lock.lock();
2259         try {
2260             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2261                     unprocessedVpnInterfaces.get(firstVpnName);
2262             if (vpnInterfaces != null) {
2263                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2264                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2265                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2266                 }
2267             } else {
2268                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2269             }
2270         } finally {
2271             lock.unlock();
2272         }
2273     }
2274
2275     public void vpnInstanceIsReady(String vpnInstanceName) {
2276         processSavedInterfaces(vpnInstanceName, true);
2277     }
2278
2279     public void vpnInstanceFailed(String vpnInstanceName) {
2280         processSavedInterfaces(vpnInstanceName, false);
2281     }
2282
2283     private static class UnprocessedVpnInterfaceData {
2284         InstanceIdentifier<VpnInterface> identifier;
2285         VpnInterface vpnInterface;
2286
2287         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2288             this.identifier = identifier;
2289             this.vpnInterface = vpnInterface;
2290         }
2291
2292         @Override
2293         public int hashCode() {
2294             final int prime = 31;
2295             int result = 1;
2296             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2297             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2298             return result;
2299         }
2300
2301         @Override
2302         public boolean equals(Object obj) {
2303             if (this == obj) {
2304                 return true;
2305             }
2306             if (obj == null) {
2307                 return false;
2308             }
2309             if (getClass() != obj.getClass()) {
2310                 return false;
2311             }
2312             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2313             if (identifier == null) {
2314                 if (other.identifier != null) {
2315                     return false;
2316                 }
2317             } else if (!identifier.equals(other.identifier)) {
2318                 return false;
2319             }
2320             if (vpnInterface == null) {
2321                 if (other.vpnInterface != null) {
2322                     return false;
2323                 }
2324             } else if (!vpnInterface.equals(other.vpnInterface)) {
2325                 return false;
2326             }
2327             return true;
2328         }
2329     }
2330
2331     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2332         String primaryRd = vpnUtil.getVpnRd(vpnName);
2333         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2334         if (vpnInstanceOpData == null) {
2335             return;
2336         }
2337         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2338         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2339             return;
2340         }
2341         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2342         vpnToDpnLists.forEach(vpnToDpnList -> {
2343             if (vpnToDpnList.getVpnInterfaces() == null) {
2344                 return;
2345             }
2346             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2347                 try {
2348                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2349                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2350                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2351                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2352                     if (!vpnInterfaceOptional.isPresent()) {
2353                         return;
2354                     }
2355                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2356                             vpnInterface.getInterfaceName());
2357                     if (configVpnAdjacencies == null) {
2358                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2359                         return;
2360                     }
2361                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2362                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2363                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2364                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2365                     // in operational DS. These unprocessed adjacencies will be handled below.
2366                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2367                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2368                     configVpnAdjacencies.stream()
2369                         .filter(adjacency -> operationVpnAdjacencies.stream()
2370                                 .noneMatch(operationalAdjacency ->
2371                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2372                         .forEach(adjacency -> {
2373                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2374                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2375                                 () -> {
2376                                     // TODO Deal with sequencing — the config tx must only submitted
2377                                     // if the oper tx goes in
2378                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2379                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2380                                         futures.add(
2381                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2382                                                 //set of prefix used, as entry in prefix-to-interface datastore
2383                                                 // is prerequisite for refresh Fib to avoid race condition leading
2384                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2385                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2386                                                 ListenableFuture<Void> configTxFuture =
2387                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2388                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2389                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2390                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2391                                                 Futures.addCallback(configTxFuture,
2392                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2393                                                     MoreExecutors.directExecutor());
2394                                                 futures.add(configTxFuture);
2395                                             }));
2396                                         return futures;
2397                                     } else {
2398                                         return emptyList();
2399                                     }
2400                                 });
2401                         });
2402                 } catch (ReadFailedException e) {
2403                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2404                             vpnName, primaryRd);
2405                 }
2406             });
2407         });
2408     }
2409
2410     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2411         private final String interfaceName;
2412         private final boolean add;
2413         private final String txnDestination;
2414
2415         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2416             this.interfaceName = interfaceName;
2417             this.add = add;
2418             this.txnDestination = transactionDest;
2419         }
2420
2421         @Override
2422         public void onSuccess(Void voidObj) {
2423             if (add) {
2424                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2425                         interfaceName, txnDestination);
2426             } else {
2427                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2428             }
2429         }
2430
2431         @Override
2432         public void onFailure(Throwable throwable) {
2433             if (add) {
2434                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2435                         interfaceName, txnDestination, throwable);
2436             } else {
2437                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2438                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2439             }
2440         }
2441     }
2442
2443     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2444         private final String primaryRd;
2445         private final Set<String> prefixListForRefreshFib;
2446
2447         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2448             this.primaryRd = primaryRd;
2449             this.prefixListForRefreshFib = prefixListForRefreshFib;
2450         }
2451
2452         @Override
2453         public void onSuccess(Void voidObj) {
2454             prefixListForRefreshFib.forEach(prefix -> {
2455                 fibManager.refreshVrfEntry(primaryRd, prefix);
2456             });
2457         }
2458
2459         @Override
2460         public void onFailure(Throwable throwable) {
2461             LOG.debug("write Tx config operation failed", throwable);
2462         }
2463     }
2464 }