vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.utils.JvmGlobalLocks;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 129
 130     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 131     private static final short DJC_MAX_RETRIES = 3;
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final IBgpManager bgpManager;
 136     private final IFibManager fibManager;
 137     private final IMdsalApiManager mdsalManager;
 138     private final IdManagerService idManager;
 139     private final OdlInterfaceRpcService ifaceMgrRpcService;
 140     private final VpnFootprintService vpnFootprintService;
 141     private final IInterfaceManager interfaceManager;
 142     private final IVpnManager vpnManager;
 143     private final ArpResponderHandler arpResponderHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final VpnUtil vpnUtil;
 146
 147     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 148
 149     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 150             new ConcurrentHashMap<>();
 151
 152     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 153
 154     @Inject
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final VpnFootprintService vpnFootprintService,
 162                                final IInterfaceManager interfaceManager,
 163                                final IVpnManager vpnManager,
 164                                final ArpResponderHandler arpResponderHandler,
 165                                final JobCoordinator jobCoordinator,
 166                                final CacheProvider cacheProvider,
 167                                final VpnUtil vpnUtil) {
 168         super(VpnInterface.class, VpnInterfaceManager.class);
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187     }
 188
 189     public Runnable isNotifyTaskQueued(String intfName) {
 190         return vpnIntfMap.remove(intfName);
 191     }
 192
 193     @PostConstruct
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204     }
 205
 206     @Override
 207     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 208         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 209     }
 210
 211     @Override
 212     protected VpnInterfaceManager getDataTreeChangeListener() {
 213         return VpnInterfaceManager.this;
 214     }
 215
 216     @Override
 217     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 218         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 219         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 220                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 221         addVpnInterface(identifier, vpnInterface, null, null);
 222     }
 223
 224     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 225                           final VpnInterface vpnInterface, String vpnName) {
 226         // FIXME: separate this out somehow?
 227         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 228         lock.lock();
 229         try {
 230             if (isVpnInstanceReady(vpnName)) {
 231                 return true;
 232             }
 233             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 234             return false;
 235         } finally {
 236             lock.unlock();
 237         }
 238     }
 239
 240     // TODO Clean up the exception handling
 241     @SuppressWarnings("checkstyle:IllegalCatch")
 242     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 244         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 245             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 246             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 247         }
 248     }
 249
 250     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 251                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 252         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 253         final String interfaceName = key.getName();
 254
 255         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 256             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 257                   vpnName, vpnInterface.getName());
 258             return;
 259         }
 260         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 261              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 262         List<Adjacency> copyOldAdjs = null;
 263         if (oldAdjs != null) {
 264             copyOldAdjs = new ArrayList<>();
 265             copyOldAdjs.addAll(oldAdjs);
 266         }
 267         List<Adjacency> copyNewAdjs = null;
 268         if (newAdjs != null) {
 269             copyNewAdjs = new ArrayList<>();
 270             copyNewAdjs.addAll(newAdjs);
 271         }
 272         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 273     }
 274
 275     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 276                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 277                     final @Nullable List<Adjacency> newAdjs,
 278                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 279         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 280         final String interfaceName = key.getName();
 281         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 282         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 283             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 284             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 285             if (interfaceState != null) {
 286                 try {
 287                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 288                     final int ifIndex = interfaceState.getIfIndex();
 289                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 290                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 291                         // (the inventory tx goes in last)
 292                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 293                         //set of prefix used, as entry in prefix-to-interface datastore
 294                         // is prerequisite for refresh Fib to avoid race condition leading to
 295                         // missing remote next hop in bucket actions on bgp-vpn delete
 296                         Set<String> prefixListForRefreshFib = new HashSet<>();
 297                         ListenableFuture<Void> confFuture =
 298                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 299                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 300                                     operTx -> futures.add(
 301                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 302                                             LOG.info(
 303                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 304                                                     + " on dpn {}",
 305                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 306                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 307                                                 confTx, operTx, invTx, interfaceState, vpnName,
 308                                                 prefixListForRefreshFib);
 309                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 310                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 311                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 312                                                 if (newAdjs != null) {
 313                                                     for (Adjacency adj : newAdjs) {
 314                                                         if (oldAdjs.contains(adj)) {
 315                                                             oldAdjs.remove(adj);
 316                                                         } else {
 317                                                             if (!isBgpVpnInternetVpn
 318                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 320                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 321                                                                     prefixListForRefreshFib);
 322                                                             }
 323                                                         }
 324                                                     }
 325                                                 }
 326                                                 for (Adjacency adj : oldAdjs) {
 327                                                     if (!isBgpVpnInternetVpn
 328                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 329                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 330                                                             operTx, confTx);
 331                                                     }
 332                                                 }
 333                                             }
 334                                         })))));
 335                         Futures.addCallback(confFuture,
 336                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 337                             MoreExecutors.directExecutor());
 338                         futures.add(confFuture);
 339                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 340                             MoreExecutors.directExecutor());
 341                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 342                             + " processed successfully", interfaceName, vpnName, dpnId);
 343                         return futures;
 344                     });
 345                 } catch (NumberFormatException | IllegalStateException e) {
 346                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 347                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 348                         vpnName, e);
 349                     return;
 350                 }
 351             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 352                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 353                     () -> {
 354                         ListenableFuture<Void> future =
 355                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 356                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 357                                     confTx, vpnName);
 358                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 359                                     vpnName, vpnInterface.getDpnId());
 360                             });
 361                         ListenableFutures.addErrorLogging(future, LOG,
 362                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 363                         return Collections.singletonList(future);
 364                     });
 365             } else {
 366                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 367                     + " is not available", interfaceName, vpnName);
 368             }
 369         } else {
 370             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 371                     + " as vpn is pending delete", interfaceName, vpnName,
 372                 vpnInterface.getDpnId());
 373         }
 374     }
 375
 376     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 377     // see comments below.
 378     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 379     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 380             final int lportTag, boolean isInterfaceUp,
 381             TypedWriteTransaction<Configuration> writeConfigTxn,
 382             TypedWriteTransaction<Operational> writeOperTxn,
 383             TypedReadWriteTransaction<Configuration> writeInvTxn,
 384             Interface interfaceState, final String vpnName,
 385             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 386         final String interfaceName = vpnInterface.getName();
 387         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 388                 vpnName);
 389         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 390         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 391         if (!isInterfaceUp) {
 392             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 393                      interfaceName, dpId, vpnName);
 394             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 395             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 396                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 397                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 398                         dpId);
 399                 return;
 400             }
 401
 402             boolean waitForVpnInterfaceOpRemoval = false;
 403             if (opVpnInterface != null) {
 404                 String opVpnName = opVpnInterface.getVpnInstanceName();
 405                 String primaryInterfaceIp = null;
 406                 if (Objects.equals(opVpnName, vpnName)) {
 407                     // Please check if the primary VRF Entry does not exist for VPNInterface
 408                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 409                     // back to back
 410                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 411                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 412                     if (adjs == null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 414                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 415                                 vpnName);
 416                         return;
 417                     }
 418                     for (Adjacency adj : adjs) {
 419                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 420                             primaryInterfaceIp = adj.getIpAddress();
 421                             break;
 422                         }
 423                     }
 424                     if (primaryInterfaceIp == null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 426                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 427                                 dpId, vpnName);
 428                         return;
 429                     }
 430                     // Get the rd of the vpn instance
 431                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 432                     if (vrf != null) {
 433                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 434                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 435                         return;
 436                     }
 437                     waitForVpnInterfaceOpRemoval = true;
 438                 } else {
 439                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 440                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 441                 }
 442             }
 443             if (!waitForVpnInterfaceOpRemoval) {
 444                 // Add the VPNInterface and quit
 445                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 446                         null/*ipAddressSourceValuePair*/,
 447                         true /* add */);
 448                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 449                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 450                 if (!isBgpVpnInternetVpn) {
 451                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 452                 }
 453                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 454                         dpId, vpnName);
 455                 if (interfaceManager.isExternalInterface(interfaceName)) {
 456                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 457                         NwConstants.ADD_FLOW);
 458                 }
 459                 return;
 460             }
 461
 462             // FIB didn't get a chance yet to clean up this VPNInterface
 463             // Let us give it a chance here !
 464             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 465                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 466             try {
 467                 Runnable notifyTask = new VpnNotifyTask();
 468                 synchronized (notifyTask) {
 469                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 470                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 471                     // published here while holding the lock on it so this path will hit the wait before notify can be
 472                     // invoked.
 473                     vpnIntfMap.put(interfaceName, notifyTask);
 474                     try {
 475                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 476                     } catch (InterruptedException e) {
 477                         // Ignored
 478                     }
 479                 }
 480             } finally {
 481                 vpnIntfMap.remove(interfaceName);
 482             }
 483
 484             if (opVpnInterface != null) {
 485                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 486                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 487                         dpId, vpnName);
 488                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 489                 return;
 490             }
 491             // VPNInterface got removed, proceed with Add
 492             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 493                     interfaceName, dpId, vpnName);
 494             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 495                     null/*ipAddressSourceValuePair*/,
 496                     true /* add */);
 497             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 498                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 499             if (!isBgpVpnInternetVpn) {
 500                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 501             }
 502             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 503                     + " FIB to clean up", interfaceName, dpId, vpnName);
 504             if (interfaceManager.isExternalInterface(interfaceName)) {
 505                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 506                                lportTag, NwConstants.ADD_FLOW);
 507             }
 508         } else {
 509             try {
 510                 // Interface is retained in the DPN, but its Link Up.
 511                 // Advertise prefixes again for this interface to BGP
 512                 InstanceIdentifier<VpnInterface> identifier =
 513                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 514                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 515                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 516                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 517                 // Perform similar operation as interface add event for extraroutes.
 518                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 519                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 520                         LogicalDatastoreType.CONFIGURATION, path);
 521                 if (!optAdjacencies.isPresent()) {
 522                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 523                     return;
 524                 }
 525                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 526                 for (Adjacency adjacency : adjacencies) {
 527                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 528                         continue;
 529                     }
 530                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 531                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 532                         continue;
 533                     }
 534                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 535                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 536                 }
 537             } catch (ReadFailedException e) {
 538                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 539                         interfaceName, vpnName, primaryRd, dpId);
 540             }
 541         }
 542     }
 543
 544     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 545         int lportTag, int addOrRemove) {
 546         Uuid extNetworkId;
 547         try {
 548             // vpn instance of ext-net interface is the network-id
 549             extNetworkId = new Uuid(vpnName);
 550         } catch (IllegalArgumentException e) {
 551             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 552                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 553             return;
 554         }
 555
 556         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 557         if (routerIds == null || routerIds.isEmpty()) {
 558             LOG.info("processExternalVpnInterface: No router is associated with {}."
 559                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 560                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 561             return;
 562         }
 563
 564         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 565                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 566         for (Uuid routerId : routerIds) {
 567             String routerName = routerId.getValue();
 568             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 569             if (Objects.equals(primarySwitch, dpId)) {
 570                 Routers router = vpnUtil.getExternalRouter(routerName);
 571                 if (router != null) {
 572                     if (addOrRemove == NwConstants.ADD_FLOW) {
 573                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 574                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 575                                 dpId, interfaceName, lportTag);
 576                     } else {
 577                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 578                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 579                                 dpId, interfaceName, lportTag);
 580                     }
 581                 } else {
 582                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 583                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 584                             interfaceName, dpId, vpnName);
 585                 }
 586             }
 587         }
 588     }
 589
 590     // TODO Clean up the exception handling
 591     @SuppressWarnings("checkstyle:IllegalCatch")
 592     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 593                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 594                                                   String vpnName, String interfaceName) {
 595         if (rd == null) {
 596             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 597                   interfaceName, dpnId, vpnName);
 598             return;
 599         } else {
 600             if (rd.equals(vpnName)) {
 601                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 602                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 603                 return;
 604             }
 605         }
 606         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 607                 interfaceName, dpnId, vpnName, rd);
 608
 609         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 610         if (nextHopIp == null) {
 611             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 612                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 613                     rd, vpnName);
 614             return;
 615         }
 616
 617         try {
 618             //Read NextHops
 619             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 620             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 621                     LogicalDatastoreType.OPERATIONAL, path);
 622             if (adjacencies.isPresent()) {
 623                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 624                 if (nextHops != null && !nextHops.isEmpty()) {
 625                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 626                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 627                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 628                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 629                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 630                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 631                     for (Adjacency nextHop : nextHops) {
 632                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 633                             continue;
 634                         }
 635                         String gatewayMac = null;
 636                         Uint32 label = Uint32.ZERO;
 637                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 638                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 639                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 640                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 641                                     .get();
 642                         } else {
 643                             label = nextHop.getLabel();
 644                         }
 645                         try {
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 647                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 648                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 649                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 650                                     gatewayMac);
 651                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 652                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 653                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 654                         } catch (Exception e) {
 655                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 656                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 657                                     interfaceName, dpnId, e);
 658                         }
 659                     }
 660                 }
 661             }
 662         } catch (ReadFailedException e) {
 663             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 664                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 665         }
 666     }
 667
 668     // TODO Clean up the exception handling
 669     @SuppressWarnings("checkstyle:IllegalCatch")
 670     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 671                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 672                         TypedWriteTransaction<Operational> writeOperTx) {
 673         //Read NextHops
 674         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 675         String rd = vpnUtil.getVpnRd(interfaceName);
 676         if (rd == null) {
 677             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 678                 interfaceName, vpnName);
 679             return;
 680         } else {
 681             if (rd.equals(vpnName)) {
 682                 LOG.info(
 683                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 684                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 685                 return;
 686             }
 687         }
 688         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 689                vpnName, rd);
 690         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 691         try {
 692             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 693                     path);
 694         } catch (ReadFailedException e) {
 695             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 696                     interfaceName, vpnName);
 697         }
 698         if (adjacencies.isPresent()) {
 699             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 700
 701             if (nextHops != null && !nextHops.isEmpty()) {
 702                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 703                         nextHops, interfaceName, vpnName, rd);
 704                 for (Adjacency nextHop : nextHops) {
 705                     try {
 706                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 707                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 708                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 709                                     interfaceName, vpnName);
 710                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 711                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 712                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 713                                     interfaceName, vpnName);
 714                         } else if (nextHop.getNextHopIpList() != null) {
 715                             // Perform similar operation as interface delete event for extraroutes.
 716                             String allocatedRd = nextHop.getVrfId();
 717                             for (String nh : nextHop.getNextHopIpList()) {
 718                                 deleteExtraRouteFromCurrentAndImportingVpns(
 719                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 720                                     writeOperTx);
 721                             }
 722                         }
 723                     } catch (Exception e) {
 724                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 725                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 726                     }
 727                 }
 728             }
 729         }
 730     }
 731
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 734                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 735                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 736                                                   TypedWriteTransaction<Operational> writeOperTxn,
 737                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 738                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 739             throws ExecutionException, InterruptedException {
 740         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 741         // Read NextHops
 742         Optional<VpnInterface> vpnInteface = Optional.absent();
 743         try {
 744             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 745             LogicalDatastoreType.CONFIGURATION, identifier);
 746         } catch (ReadFailedException e) {
 747             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 748                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 749         }
 750         Uuid intfnetworkUuid = null;
 751         NetworkType networkType = null;
 752         Long segmentationId = Long.valueOf(-1);
 753         Adjacencies adjacencies = null;
 754         if (vpnInteface.isPresent()) {
 755             intfnetworkUuid = vpnInteface.get().getNetworkId();
 756             networkType = vpnInteface.get().getNetworkType();
 757             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 758             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 759             if (adjacencies == null) {
 760                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 761                         null/*gwMac*/,  null/*gatewayIp*/, writeOperTxn);
 762                 return;
 763             }
 764         }
 765         // Get the rd of the vpn instance
 766         String nextHopIp = null;
 767         String gatewayIp = null;
 768         try {
 769             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 770         } catch (Exception e) {
 771             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 772                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 773         }
 774         List<String> nhList = new ArrayList<>();
 775         if (nextHopIp != null) {
 776             nhList.add(nextHopIp);
 777             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 778                     interfaceName, dpnId, vpnName, nhList);
 779         }
 780         Optional<String> gwMac = Optional.absent();
 781         String vpnInterfaceSubnetGwMacAddress = null;
 782         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 783         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 784         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 785         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 786         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 787         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 788         List<Adjacency> value = new ArrayList<>();
 789         for (Adjacency nextHop : nextHops) {
 790             String rd = primaryRd;
 791             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 792             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 793                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 794                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 795                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 796                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 797                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 798                 continue;
 799             }
 800             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 801                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 802                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 803                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 804                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 805                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 806
 807                 Prefixes prefixes = intfnetworkUuid != null
 808                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 809                             segmentationId, prefixCue) :
 810                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 811                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 812                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 813                 final Uuid subnetId = nextHop.getSubnetId();
 814
 815                 gatewayIp = nextHop.getSubnetGatewayIp();
 816                 if (gatewayIp == null) {
 817                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 818                     if (gatewayIpOptional.isPresent()) {
 819                         gatewayIp = gatewayIpOptional.get();
 820                     }
 821                 }
 822
 823                 if (gatewayIp != null) {
 824                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 825                     if (gwMac.isPresent()) {
 826                         // A valid mac-address is available for this subnet-gateway-ip
 827                         // Use this for programming ARP_RESPONDER table here.  And save this
 828                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 829                         // to populate L3_GW_MAC_TABLE there.
 830                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 831                                 gatewayIp, gwMac.get());
 832                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 833                     } else {
 834                         // A valid mac-address is not available for this subnet-gateway-ip
 835                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 836                         // Save this connected-mac-address as gateway-mac-address for the
 837                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 838                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 839                         if (gwMac.isPresent()) {
 840                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 841                                     NwConstants.ADD_FLOW, gwMac.get());
 842                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 843                                     gatewayIp, gwMac.get());
 844                         } else {
 845                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 846                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 847                                 + "gwIp {}",
 848                                 subnetId, interfaceName, vpnName, gatewayIp);
 849                         }
 850                     }
 851                 } else {
 852                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 853                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 854                         subnetId, interfaceName, vpnName);
 855                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 856                 }
 857                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 858                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 859             } else {
 860                 //Extra route adjacency
 861                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 862                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 863                 // FIXME: separate this out somehow?
 864                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 865                 lock.lock();
 866                 try {
 867                     java.util.Optional<String> rdToAllocate = vpnUtil
 868                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 869                                     nextHop.getNextHopIpList().get(0), dpnId);
 870                     if (rdToAllocate.isPresent()) {
 871                         rd = rdToAllocate.get();
 872                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 873                             rd, prefix);
 874                     } else {
 875                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 876                         continue;
 877                     }
 878                 } finally {
 879                     lock.unlock();
 880                 }
 881                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 882                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 883                         interfaceName, dpnId);
 884             }
 885             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 886             // can be different from the gateway-mac-address within the VRFEntry as the
 887             // gateway-mac-address is a superset.
 888             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 889             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 890                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 891                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 892             Adjacency operationalAdjacency = null;
 893             try {
 894                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 895             } catch (NullPointerException e) {
 896                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 897                     input, e.getMessage());
 898                 return;
 899             }
 900             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 901                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 902                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 903                     writeConfigTxn);
 904             }
 905             value.add(operationalAdjacency);
 906         }
 907
 908         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 909         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 910                 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
 911
 912         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 913                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 914                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 915
 916         for (Adjacency nextHop : aug.getAdjacency()) {
 917             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 918             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 919                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 920                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 921                 registeredPopulator.populateFib(input, writeConfigTxn);
 922             }
 923         }
 924     }
 925
 926     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 927                                               long lportTag, String gwMac, String gwIp,
 928                                               TypedWriteTransaction<Operational> writeOperTxn) {
 929         VpnInterfaceOpDataEntry opInterface =
 930               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
 931         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 932             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 933         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 934         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 935                 interfaceName, dpnId, vpnName);
 936     }
 937
 938     // TODO Clean up the exception handling
 939     @SuppressWarnings("checkstyle:IllegalCatch")
 940     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 941                                            StateTunnelList stateTunnelList,
 942                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 943                                            TypedWriteTransaction<Operational> writeOperTxn) {
 944
 945         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 946         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 947         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 948         List<Adjacency> adjList =
 949             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 950         if (adjList.isEmpty()) {
 951             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 952                     vpnInterface, srcDpnId);
 953             return;
 954         }
 955         String prefix = null;
 956         List<Adjacency> value = new ArrayList<>();
 957         boolean isNextHopAddReqd = false;
 958         String vpnName = vpnInterface.getVpnInstanceName();
 959         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 960         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 961         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 962                 vpnInterface.getName(), vpnInterface.getDpnId(),
 963                 vpnInterface.getVpnInstanceName(), adjList);
 964         for (Adjacency adj : adjList) {
 965             String rd = adj.getVrfId();
 966             rd = rd != null ? rd : vpnName;
 967             prefix = adj.getIpAddress();
 968             Uint32 label = adj.getLabel();
 969             List<String> nhList = Collections.singletonList(srcTepIp);
 970             List<String> nextHopList = adj.getNextHopIpList();
 971             // If TEP is added , update the nexthop of primary adjacency.
 972             // Secondary adj nexthop is already pointing to primary adj IP address.
 973             if (nextHopList == null || nextHopList.isEmpty()) {
 974                 isNextHopAddReqd = true;
 975             }
 976
 977             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 978                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 979             } else {
 980                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 981                 if (!vrfEntryOptional.isPresent()) {
 982                     continue;
 983                 }
 984                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 985                 if (!nhList.contains(srcTepIp)) {
 986                     nhList.add(srcTepIp);
 987                     isNextHopAddReqd = true;
 988                 }
 989                 value.add(adj);
 990             }
 991
 992             if (isNextHopAddReqd) {
 993                 updateLabelMapper(label, nhList);
 994                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 995                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 996                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 997                 // Update the VRF entry with nextHop
 998                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 999                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1000
1001                 //Get the list of VPN's importing this route(prefix) .
1002                 // Then update the VRF entry with nhList
1003                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1004                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1005                     String vpnRd = vpn.getVrfId();
1006                     if (vpnRd != null) {
1007                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1008                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1009                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1010                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1011                             vpnInterface.getName(), srcDpnId, vpnName,
1012                             vpn.getVpnInstanceName(), vpnRd);
1013                     }
1014                 }
1015                 // Advertise the prefix to BGP only for external vpn
1016                 // since there is a nexthop change.
1017                 try {
1018                     if (!rd.equalsIgnoreCase(vpnName)) {
1019                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1020                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1021                                 null /*gatewayMacAddress*/);
1022                     }
1023                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1024                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1025                             srcDpnId, vpnName);
1026                 } catch (Exception ex) {
1027                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1028                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1029                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1030                 }
1031             }
1032         }
1033         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1034         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1035                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1036                 .addAugmentation(AdjacenciesOp.class, aug).build();
1037         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1038                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1039         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1040         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1041                 vpnInterface.getName(), srcDpnId, vpnName);
1042
1043     }
1044
1045     // TODO Clean up the exception handling
1046     @SuppressWarnings("checkstyle:IllegalCatch")
1047     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1048                                               StateTunnelList stateTunnelList,
1049                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1050                                               TypedWriteTransaction<Operational> writeOperTxn) {
1051
1052         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1053         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1054         String prefix = null;
1055         boolean isNextHopRemoveReqd = false;
1056         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1057         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1058         String vpnName = vpnInterface.getVpnInstanceName();
1059         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1060         String primaryRd = vpnUtil.getVpnRd(vpnName);
1061         if (adjList != null) {
1062             List<Adjacency> value = new ArrayList<>();
1063             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1064                     vpnInterface.getName(), vpnInterface.getDpnId(),
1065                     vpnInterface.getVpnInstanceName(), adjList);
1066             for (Adjacency adj : adjList) {
1067                 List<String> nhList = new ArrayList<>();
1068                 String rd = adj.getVrfId();
1069                 rd = rd != null ? rd : vpnName;
1070                 prefix = adj.getIpAddress();
1071                 List<String> nextHopList = adj.getNextHopIpList();
1072                 Uint32 label = adj.getLabel();
1073                 if (nextHopList != null && !nextHopList.isEmpty()) {
1074                     isNextHopRemoveReqd = true;
1075                 }
1076                 // If TEP is deleted , remove the nexthop from primary adjacency.
1077                 // Secondary adj nexthop will continue to point to primary adj IP address.
1078                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1079                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1080                 } else {
1081                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1082                     if (!vrfEntryOptional.isPresent()) {
1083                         continue;
1084                     }
1085                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1086                     if (nhList.contains(srcTepIp)) {
1087                         nhList.remove(srcTepIp);
1088                         isNextHopRemoveReqd = true;
1089                     }
1090                     value.add(adj);
1091                 }
1092
1093                 if (isNextHopRemoveReqd) {
1094                     updateLabelMapper(label, nhList);
1095                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1096                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1097                             prefix, nhList, vpnName,
1098                             vpnId, rd, vpnInterface.getName());
1099                     // Update the VRF entry with removed nextHop
1100                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1101                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1102
1103                     //Get the list of VPN's importing this route(prefix) .
1104                     // Then update the VRF entry with nhList
1105                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1106                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1107                         String vpnRd = vpn.getVrfId();
1108                         if (vpnRd != null) {
1109                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1110                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1111                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1112                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1113                                     nhList, label, vpnInterface.getName(), srcDpnId,
1114                                     vpnName,
1115                                     vpn.getVpnInstanceName(), vpnRd);
1116                         }
1117                     }
1118
1119                     // Withdraw prefix from BGP only for external vpn.
1120                     try {
1121                         if (!rd.equalsIgnoreCase(vpnName)) {
1122                             bgpManager.withdrawPrefix(rd, prefix);
1123                         }
1124                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1125                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1126                                 vpnInterface.getName(), srcDpnId,
1127                                 vpnName);
1128                     } catch (Exception ex) {
1129                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1130                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1131                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1132                     }
1133                 }
1134             }
1135             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1136             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1137                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1138                     .addAugmentation(AdjacenciesOp.class, aug).build();
1139             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1140                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1141             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1142             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1143                          vpnInterface.getName(), srcDpnId, vpnName);
1144         }
1145     }
1146
1147     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1148         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1149
1150         String vpnRd = vpnUtil.getVpnRd(vpnName);
1151         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1152         if (vpnInstanceOpDataEntry == null) {
1153             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1154                     + " to check for vpns exporting the routes", vpnName);
1155             return vpnsToExportRoute;
1156         }
1157
1158         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1159             if (input.getVpnInstanceName() == null) {
1160                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1161                         input.getVrfId());
1162                 return false;
1163             }
1164             return !input.getVpnInstanceName().equals(vpnName);
1165         };
1166
1167         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1168             Iterable<String> commonRTs =
1169                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1170                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1171             return Iterators.size(commonRTs.iterator()) > 0;
1172         };
1173
1174         vpnsToExportRoute =
1175                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1176                         Collectors.toList());
1177         return vpnsToExportRoute;
1178     }
1179
1180     // TODO Clean up the exception handling
1181     @SuppressWarnings("checkstyle:IllegalCatch")
1182     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1183         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1184         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1185             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1186             if (vrfEntries != null) {
1187                 ListenableFutures.addErrorLogging(
1188                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1189                         for (VrfEntry vrfEntry : vrfEntries) {
1190                             try {
1191                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1192                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1193                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1194                                             + " is not a controller managed non intervpn link route. Ignoring.",
1195                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1196                                     continue;
1197                                 }
1198                                 String prefix = vrfEntry.getDestPrefix();
1199                                 String gwMac = vrfEntry.getGatewayMacAddress();
1200                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1201                                     String nh = routePath.getNexthopAddress();
1202                                     Uint32 label = routePath.getLabel();
1203                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1204                                         vrfEntry.getOrigin()))) {
1205                                         LOG.info(
1206                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1207                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1208                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1209                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1210                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1211                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1212                                             confTx);
1213                                     } else {
1214                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1215                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1216                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1217                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1218                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1219                                             confTx);
1220                                     }
1221                                 });
1222                             } catch (RuntimeException e) {
1223                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1224                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1225                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1226                             }
1227                         }
1228                     }), LOG, "Error handing VPN exporting routes");
1229             } else {
1230                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1231                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1232             }
1233         }
1234     }
1235
1236     @Override
1237     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1238         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1239         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1240         final String interfaceName = key.getName();
1241         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1242             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1243             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1244         }
1245     }
1246
1247     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1248                                 final VpnInterface vpnInterface, final String vpnName,
1249                                 final String interfaceName) {
1250         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1251             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1252                 ListenableFuture<Void> future =
1253                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1254                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1255                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1256                     });
1257                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1258                         vpnInterface.getName(), vpnName);
1259                 return Collections.singletonList(future);
1260             }, DJC_MAX_RETRIES);
1261         } else {
1262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1263             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1264         }
1265     }
1266
1267     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1268     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1269                                            final VpnInterface vpnInterface, final String vpnName,
1270                                            final String interfaceName, final Interface interfaceState) {
1271         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1272                 vpnName, vpnInterface.getDpnId());
1273         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1274         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1275             () -> {
1276                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1277                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1278                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1279                         writeOperTxn -> futures.add(
1280                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1281                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1282                                         interfaceName, vpnName);
1283                                 Uint64 dpId;
1284                                 int ifIndex;
1285                                 String gwMacAddress;
1286                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1287                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1288                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1289                                 try {
1290                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1291                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1292                                 } catch (ReadFailedException e) {
1293                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1294                                             interfaceName, vpnName);
1295                                     return;
1296                                 }
1297                                 if (interfaceState != null) {
1298                                     try {
1299                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1300                                     } catch (NumberFormatException | IllegalStateException e) {
1301                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1302                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1303                                                         + " from vpn interface op data store. ", interfaceName,
1304                                                 vpnInterface.getDpnId(), vpnName, e);
1305                                         dpId = Uint64.ZERO;
1306                                     }
1307                                     ifIndex = interfaceState.getIfIndex();
1308                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1309                                 } else {
1310                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1311                                             + " from vpn interface op.", interfaceName);
1312                                     if (optVpnInterface.isPresent()) {
1313                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1314                                         dpId = vpnOpInterface.getDpnId();
1315                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1316                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1317                                     } else {
1318                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1319                                                 + " as interfaceState and vpn interface op is not"
1320                                                 + " available", interfaceName, vpnName);
1321                                         return;
1322                                     }
1323                                 }
1324                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1325                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1326                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1327                                 LOG.info(
1328                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1329                                                 + "successfully",
1330                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1331                             })))));
1332                 futures.add(configFuture);
1333                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1334                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1335                 return futures;
1336             }, DJC_MAX_RETRIES);
1337     }
1338
1339     protected void processVpnInterfaceDown(Uint64 dpId,
1340                                            String interfaceName,
1341                                            int lportTag,
1342                                            String gwMac,
1343                                            VpnInterfaceOpDataEntry vpnOpInterface,
1344                                            boolean isInterfaceStateDown,
1345                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1346                                            TypedWriteTransaction<Operational> writeOperTxn,
1347                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1348             throws ExecutionException, InterruptedException {
1349         if (vpnOpInterface == null) {
1350             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1351                     + " as it is not available in operational data store", interfaceName, dpId);
1352             return;
1353         }
1354         final String vpnName = vpnOpInterface.getVpnInstanceName();
1355         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1356                                                     interfaceName, vpnName);
1357         if (!isInterfaceStateDown) {
1358             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1359             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1360             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1361             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1362                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1363             if (interfaceManager.isExternalInterface(interfaceName)) {
1364                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1365                     NwConstants.DEL_FLOW);
1366             }
1367             if (!isBgpVpnInternetVpn) {
1368                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1369             }
1370             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1371                     + " successful", interfaceName, dpId, vpnName);
1372         } else {
1373             // Interface is retained in the DPN, but its Link Down.
1374             // Only withdraw the prefixes for this interface from BGP
1375             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1376         }
1377     }
1378
1379     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1380                                           final String vpnName, final Uint32 vpnId, String gwMac,
1381                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1382                                           TypedWriteTransaction<Operational> writeOperTxn,
1383                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1384             throws ExecutionException, InterruptedException {
1385         //Read NextHops
1386         try {
1387             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1388                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1389             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1390                     SingleTransactionDataBroker.syncReadOptional(dataBroker,
1391                             LogicalDatastoreType.OPERATIONAL, identifier);
1392             boolean isLearntIP = Boolean.FALSE;
1393             String primaryRd = vpnUtil.getVpnRd(vpnName);
1394             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1395                     interfaceName, dpnId, vpnName, primaryRd);
1396             if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1397                 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1398                         + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1399                 return;
1400             }
1401             AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1402
1403             if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) {
1404                 List<Adjacency> nextHops = adjacencies.getAdjacency();
1405                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1406                         interfaceName, dpnId, vpnName, nextHops);
1407                 for (Adjacency nextHop : nextHops) {
1408                     if (nextHop.isPhysNetworkFunc()) {
1409                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1410                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1411                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1412                                 null/*writeCfgTxn*/);
1413                     } else {
1414                         String rd = nextHop.getVrfId();
1415                         List<String> nhList;
1416                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1417                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1418                             isLearntIP = nextHop.getAdjacencyType() == AdjacencyType.LearntIp ? true : false;
1419                         } else {
1420                             // This is a primary adjacency
1421                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1422                                     : emptyList();
1423                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1424                                     vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1425                                     interfaceName, writeInvTxn);
1426                         }
1427                         if (!nhList.isEmpty()) {
1428                             if (Objects.equals(primaryRd, vpnName)) {
1429                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1430                                 //remove from FIB directly
1431                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1432                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1433                             } else {
1434                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1435                                         interfaceName, writeConfigTxn, writeOperTxn);
1436                             }
1437                         } else {
1438                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1439                                             + " interface {}", nextHop.getIpAddress(), rd,
1440                                     nextHop.getAdjacencyType().toString(), interfaceName);
1441                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1442                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1443                         }
1444                     }
1445                     String ip = nextHop.getIpAddress().split("/")[0];
1446                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1447                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1448                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1449                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1450                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1451                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1452                     }
1453                     // Remove the MIP-IP from VpnPortIpToPort.
1454                     if (isLearntIP) {
1455                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1456                         if (persistedIp != null && persistedIp.isLearntIp()
1457                                 && persistedIp.getPortName().equals(interfaceName)) {
1458                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1459                             LOG.info(
1460                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1461                                             + "from VpnPortipToPort",
1462                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1463                         }
1464                     }
1465                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1466                     if (vpnPortipToPort != null) {
1467                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1468                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1469                                  + "Interface {} ip {} on dpn {} for vpn {}",
1470                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1471                     }
1472                 }
1473             } else {
1474                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1475                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1476                         + " Removing it.", interfaceName, vpnName, dpnId);
1477                 writeOperTxn.delete(identifier);
1478             }
1479         } catch (ReadFailedException e) {
1480             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1481                     interfaceName, dpnId, vpnName);
1482         }
1483     }
1484
1485     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1486                                                             String interfaceName, Uint64 dpnId,
1487                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1488                                                             TypedWriteTransaction<Operational> writeOperTx) {
1489         return (nh) -> {
1490             String primaryRd = vpnUtil.getVpnRd(vpnName);
1491             String prefix = nextHop.getIpAddress();
1492             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1493             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1494                     nextHop, vpnName, interfaceName, dpnId);
1495             // FIXME: separate this out somehow?
1496             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1497             lock.lock();
1498             try {
1499                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1500                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1501                     //If extra-route is present behind at least one VM, then do not remove or update
1502                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1503                     //prefixtointerface DS
1504                     return;
1505                 }
1506                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1507                         writeConfigTxn);
1508             } finally {
1509                 lock.unlock();
1510             }
1511             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1512                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1513                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1514         };
1515     }
1516
1517     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1518                                            Uint64 dpnId, String rd, String interfaceName,
1519                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1520                                            TypedWriteTransaction<Operational> writeOperTx) {
1521         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1522                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1523         nhList.forEach((nh) -> {
1524             //IRT: remove routes from other vpns importing it
1525             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1526                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1527             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1528                 String vpnRd = vpn.getVrfId();
1529                 if (vpnRd != null) {
1530                     fibManager.removeOrUpdateFibEntry(vpnRd,
1531                             nextHop.getIpAddress(), nh, writeConfigTxn);
1532                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1533                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1534                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1535                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1536                 }
1537             }
1538         });
1539     }
1540
1541     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1542                                                  int lportTag, String gwMac, String gwIp, String interfaceName,
1543                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1544             throws ExecutionException, InterruptedException {
1545         final Uuid subnetId = nextHop.getSubnetId();
1546         if (nextHop.getSubnetGatewayMacAddress() == null) {
1547             // A valid mac-address was not available for this subnet-gateway-ip
1548             // So a connected-mac-address was used for this subnet and we need
1549             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1550             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1551         }
1552         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1553                 subnetId);
1554     }
1555
1556     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1557                                                   String interfaceName) {
1558         // This is either an extra-route (or) a learned IP via subnet-route
1559         List<String> nhList = null;
1560         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1561         if (nextHopIp == null || nextHopIp.isEmpty()) {
1562             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1563                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1564                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1565                     vpnName);
1566             nhList = emptyList();
1567         } else {
1568             nhList = Collections.singletonList(nextHopIp);
1569         }
1570         return nhList;
1571     }
1572
1573     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1574         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1575         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1576         // else use connected interface
1577         if (gwPort != null && gwPort.isSubnetIp()) {
1578             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1579                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1580             return Optional.of(gwPort.getMacAddress());
1581         }
1582         return Optional.absent();
1583     }
1584
1585     @Override
1586     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1587         final VpnInterface update) {
1588         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1589         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1590                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1591         if (original.equals(update)) {
1592             LOG.info("update: original {} update {} are same. No update required.", original, update);
1593             return;
1594         }
1595         final String vpnInterfaceName = update.getName();
1596         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1597         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1598         //handles switching between <internal VPN - external VPN>
1599         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1600             List<ListenableFuture<Void>> futures = new ArrayList<>();
1601             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1602                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1603                                 + "to newVpn(s) {}",
1604                         original.getName(), dpnId,
1605                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1606                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1607                 return emptyList();
1608             }
1609             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1610             return futures;
1611         });
1612     }
1613
1614     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1615                                                            VpnInterface original, VpnInterface update,
1616                                                            List<ListenableFuture<Void>> futures) {
1617         boolean isVpnInstanceUpdate = false;
1618         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1619         final String interfaceName = key.getName();
1620         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1621         List<String> oldVpnListCopy = new ArrayList<>();
1622         oldVpnListCopy.addAll(oldVpnList);
1623         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1624         List<String> newVpnListCopy = new ArrayList<>();
1625         newVpnListCopy.addAll(newVpnList);
1626
1627         oldVpnList.removeAll(newVpnList);
1628         newVpnList.removeAll(oldVpnListCopy);
1629         //This block will execute only on if there is a change in the VPN Instance.
1630         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1631             /*
1632              * Internet BGP-VPN Instance update with single router:
1633              * ====================================================
1634              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1635              *     1st VPN Instance : router VPN or external BGP-VPN.
1636              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1637              *
1638              * VPN Instance UPDATE:
1639              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1640              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1641              *
1642              * External BGP-VPN Instance update with single router:
1643              * ====================================================
1644              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1645              *
1646              * Updated VPN Instance will be always either internal router VPN to
1647              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1648              *
1649              * VPN Instance UPDATE:
1650              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1651              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1652              *
1653              * Dual Router VPN Instance Update:
1654              * ================================
1655              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1656              *
1657              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1658              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1659              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1660              *
1661              * Dual Router --> Associated with common external BGP-VPN Instance.
1662              * 1st router and 2nd router are getting associated with single External BGP-VPN
1663              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1664              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1665              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1666              *
1667              *
1668              */
1669             isVpnInstanceUpdate = true;
1670             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1671                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1672                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1673                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1674                     List<String> externalBgpVpnList = new ArrayList<>();
1675                     for (String newVpnName : newVpnListCopy) {
1676                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1677                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1678                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1679                                 .BgpvpnType.BGPVPNExternal) {
1680                             externalBgpVpnList.add(newVpnName);
1681                             break;
1682                         }
1683                     }
1684                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1685                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1686                             externalBgpVpnList, oldVpnListCopy, futures);
1687
1688                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1689                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1690                     //Identify the router VPN Instance and pass that value as oldVpnList
1691                     List<String> routerVpnList = new ArrayList<>();
1692                     for (String newVpnName : newVpnListCopy) {
1693                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1694                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1695                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1696                                 .BgpvpnType.VPN) {
1697                             routerVpnList.add(newVpnName);
1698                             break;
1699                         }
1700                     }
1701                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1702                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1703                             newVpnList, oldVpnListCopy, futures);
1704
1705                 } else {
1706                     //Handle remaining use cases.
1707                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1708                             oldVpnListCopy, futures);
1709                 }
1710             } else {
1711                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1712                         oldVpnListCopy, futures);
1713             }
1714         }
1715         return isVpnInstanceUpdate;
1716     }
1717
1718     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1719                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1720                                          List<String> newVpnList, List<String> oldVpnListCopy,
1721                                          List<ListenableFuture<Void>> futures) {
1722         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1723         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1724                 ? origAdjs.getAdjacency() : new ArrayList<>();
1725         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1726         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1727                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1728
1729         boolean isOldVpnRemoveCallExecuted = false;
1730         for (String oldVpnName : oldVpnList) {
1731             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1732                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1733             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1734             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1735                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1736                     newVpnList);
1737             isOldVpnRemoveCallExecuted = true;
1738         }
1739         //Wait for previous interface bindings to be removed
1740         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1741             try {
1742                 Thread.sleep(2000);
1743             } catch (InterruptedException e) {
1744                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1745             }
1746         }
1747         for (String newVpnName : newVpnList) {
1748             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1749             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1750                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1751                         + "onto vpnName {} ", interfaceName, newVpnName);
1752                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1753                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1754                                 + "from oldVpn(s) {} to newVpn {} ",
1755                         interfaceName, oldVpnListCopy, newVpnName);
1756                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1757                  * Use Case:
1758                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1759                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1760                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1761                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1762                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1763                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1764                  *     router VPN as well as Internet BGP-VPN.
1765                  *
1766                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1767                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1768                  *                                       update in existing router VPN instance.
1769                  */
1770                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1771                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1772                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1773                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1774                 }
1775             } else {
1776                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1777                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1778                         interfaceName, oldVpnListCopy, newVpnName);
1779             }
1780         }
1781     }
1782
1783     // TODO Clean up the exception handling
1784     @SuppressWarnings("checkstyle:IllegalCatch")
1785     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1786                                                                     String vpnInterfaceName,
1787                                                                     List<ListenableFuture<Void>> futures) {
1788         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1789         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1790                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1791         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1792         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1793                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1794
1795         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1796         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1797             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1798             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1799             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1800             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1801             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1802                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1803                 //set of prefix used as entry in prefix-to-interface datastore
1804                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1805                 // in bucket actions on bgp-vpn delete
1806                 Set<String> prefixListForRefreshFib = new HashSet<>();
1807                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1808                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1809                         operTx -> {
1810                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1811                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1812                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1813                                     update.getName(), newVpnName);
1814                             //handle both addition and removal of adjacencies
1815                             // currently, new adjacency may be an extra route
1816                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1817                             if (!oldAdjs.equals(newAdjs)) {
1818                                 for (Adjacency adj : copyNewAdjs) {
1819                                     if (copyOldAdjs.contains(adj)) {
1820                                         copyOldAdjs.remove(adj);
1821                                     } else {
1822                                         // add new adjacency
1823                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1824                                             try {
1825                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1826                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1827                                             } catch (RuntimeException e) {
1828                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1829                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1830                                             }
1831                                         }
1832                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1833                                             + " added to vpn interface {} on vpn {} dpnId {}",
1834                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1835                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1836                                     }
1837                                 }
1838                                 for (Adjacency adj : copyOldAdjs) {
1839                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1840                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1841                                             && !adj.isPhysNetworkFunc()) {
1842                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1843                                                 confTx);
1844                                             //remove FIB entry
1845                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1846                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1847                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1848                                             //remove BGP entry
1849                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1850                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1851                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1852                                             }
1853                                         } else {
1854                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1855                                                 operTx, confTx);
1856                                         }
1857                                     }
1858                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1859                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1860                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1861                                 }
1862                             }
1863                         })));
1864                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1865                     MoreExecutors.directExecutor());
1866                 futures.add(configTxFuture);
1867                 for (ListenableFuture<Void> future : futures) {
1868                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1869                             update.getName(), update.getVpnInstanceNames());
1870                 }
1871             } else {
1872                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1873                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1874             }
1875         }
1876         return futures;
1877     }
1878
1879     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1880         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1881                 .toString();
1882         // FIXME: separate this out somehow?
1883         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1884         lock.lock();
1885         try {
1886             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1887                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1888             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1889                 LogicalDatastoreType.OPERATIONAL, lriIid);
1890             if (opResult.isPresent()) {
1891                 LabelRouteInfo labelRouteInfo =
1892                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1893                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1894                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1895             }
1896             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1897                     nextHopIpList);
1898         } catch (ReadFailedException e) {
1899             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1900                     nextHopIpList);
1901         } catch (TransactionCommitFailedException e) {
1902             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1903                     nextHopIpList);
1904         } finally {
1905             lock.unlock();
1906         }
1907     }
1908
1909     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1910         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1911
1912         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1913         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1914                 .addAugmentation(SubnetRoute.class, route).build();
1915         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1916         InstanceIdentifierBuilder<VrfTables> idBuilder =
1917             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1918         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1919         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1920         if (writeConfigTxn != null) {
1921             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1922         } else {
1923             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1924         }
1925         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1926                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1927     }
1928
1929     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1930                                            Adjacency adj, Uint64 dpnId,
1931                                            TypedWriteTransaction<Operational> writeOperTxn,
1932                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1933                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1934                                            Set<String> prefixListForRefreshFib)
1935             throws ExecutionException, InterruptedException {
1936         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1937         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1938         try {
1939             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1940                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1941             if (optVpnInterface.isPresent()) {
1942                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1943                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1944                 String vpnName = currVpnIntf.getVpnInstanceName();
1945                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1946                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1947                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1948                         LogicalDatastoreType.OPERATIONAL, adjPath);
1949                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1950                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1951                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1952                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1953                 List<Adjacency> adjacencies = new ArrayList<>();
1954                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1955                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1956                 }
1957                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1958                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1959                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1960                 Adjacency operationalAdjacency = null;
1961                 //Handling dual stack neutron port primary adjacency
1962                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1963                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1964                             currVpnIntf.getName(), vpnName);
1965                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1966                             currVpnIntf.getName());
1967                     if (interfaceState != null) {
1968                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1969                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1970                             prefixListForRefreshFib);
1971                     }
1972                 }
1973                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1974                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1975                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1976                             : RouteOrigin.STATIC;
1977                     String nh = adj.getNextHopIpList().get(0);
1978                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1979                     // FIXME: separate out to somehow?
1980                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1981                     lock.lock();
1982                     try {
1983                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1984                                 vpnId, null, prefix, vpnName, nh, dpnId);
1985                         if (rdToAllocate.isPresent()) {
1986                             input.setRd(rdToAllocate.get());
1987                             operationalAdjacency = populator.createOperationalAdjacency(input);
1988                             int label = operationalAdjacency.getLabel().intValue();
1989                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1990                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1991                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1992                                     prefixListForRefreshFib, writeConfigTxn);
1993                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1994                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1995                                     vpnName, label, currVpnIntf.getName(), dpnId);
1996                         } else {
1997                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1998                                     vpnName, prefix);
1999                             return;
2000                         }
2001                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2002                         // Keeping the MPLS check for now.
2003                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2004                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2005                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2006                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
2007                             vpnsToImportRoute.forEach(vpn -> {
2008                                 if (vpn.getVrfId() != null) {
2009                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2010                                             vpnId, prefix,
2011                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2012                                             .ifPresent(
2013                                                 rds -> vpnManager.addExtraRoute(
2014                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2015                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2016                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2017                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2018                                 }
2019                             });
2020                         }
2021                     } finally {
2022                         lock.unlock();
2023                     }
2024                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2025                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2026                             currVpnIntf.getName(), vpnName);
2027
2028                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2029                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2030                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2031                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2032                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2033                             Prefixes.PrefixCue.PhysNetFunc);
2034                     if (vpnIntefaceConfig.isPresent()) {
2035                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2036                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2037                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2038                     }
2039
2040                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2041
2042                     writeOperTxn.merge(
2043                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2044                                     prefix), pnfPrefix, true);
2045
2046                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2047                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2048                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2049                             RouteOrigin.LOCAL, writeConfigTxn);
2050
2051                     input.setRd(adj.getVrfId());
2052                 }
2053                 if (operationalAdjacency == null) {
2054                     operationalAdjacency = populator.createOperationalAdjacency(input);
2055                 }
2056                 adjacencies.add(operationalAdjacency);
2057                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2058                 VpnInterfaceOpDataEntry newVpnIntf =
2059                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2060                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2061                                 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2062                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2063             }
2064         } catch (ReadFailedException e) {
2065             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2066                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2067         }
2068     }
2069
2070     @Nullable
2071     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2072         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2073         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2074     }
2075
2076     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2077                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2078                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2079         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2080         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2081         try {
2082             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2083                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2084             if (optVpnInterface.isPresent()) {
2085                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2086                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2087                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2088                         LogicalDatastoreType.OPERATIONAL, path);
2089                 if (optAdjacencies.isPresent()) {
2090                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2091
2092                     if (adjacencies != null && !adjacencies.isEmpty()) {
2093                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2094                         for (Adjacency adjacency : adjacencies) {
2095                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2096                                 String rd = adjacency.getVrfId();
2097                                 if (adj.getNextHopIpList() != null) {
2098                                     for (String nh : adj.getNextHopIpList()) {
2099                                         deleteExtraRouteFromCurrentAndImportingVpns(
2100                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2101                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2102                                     }
2103                                 } else if (adj.isPhysNetworkFunc()) {
2104                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2105                                             adj.getIpAddress(), adj.getSubnetId());
2106                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2107                                             writeConfigTxn);
2108                                 }
2109                                 break;
2110                             }
2111
2112                         }
2113                     }
2114                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2115                             dpnId, adj.getVrfId());
2116                 } else {
2117                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2118                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2119                 }
2120             }
2121         } catch (ReadFailedException e) {
2122             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2123                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2124         }
2125     }
2126
2127     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2128                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2129                                     TypedWriteTransaction<Operational> writeOperTx) {
2130         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2131                 destination, nextHop, vpnName, intfName, rd);
2132         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2133         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2134         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2135             String vpnRd = vpn.getVrfId();
2136             if (vpnRd != null) {
2137                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2138                         destination, nextHop, vpnName, intfName, vpnRd);
2139                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2140                         writeOperTx);
2141             }
2142         }
2143     }
2144
2145     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2146         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2147             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2148             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2149     }
2150
2151     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2152         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2153             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2154     }
2155
2156     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2157                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2158         if (vpnInterface == null) {
2159             return;
2160         }
2161         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2162         if (adjs == null) {
2163             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2164                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2165             return;
2166         }
2167         for (Adjacency adj : adjs) {
2168             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2169                 String primaryInterfaceIp = adj.getIpAddress();
2170                 String macAddress = adj.getMacAddress();
2171                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2172
2173                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2174                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2175
2176                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2177                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2178                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2179                         routerInt, label, writeConfigTxn);
2180                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2181                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2182                         macAddress);
2183             } else {
2184                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2185                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2186                         interfaceName, primaryRd, vpnName);
2187             }
2188         }
2189     }
2190
2191     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2192             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2193         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2194         String rd = vpnUtil.getVpnRd(vpnName);
2195         if (adjs != null) {
2196             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2197             for (Adjacency adj : adjsList) {
2198                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2199                     String primaryInterfaceIp = adj.getIpAddress();
2200                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2201                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2202                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2203                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2204                 }
2205             }
2206         } else {
2207             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2208                     vpnInterface.getName(), rd);
2209         }
2210     }
2211
2212     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2213         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2214         final String interfaceName = key.getName();
2215         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2216                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2217         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2218                   intefaceData.identifier, vpnName);
2219     }
2220
2221     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2222                                               VpnInterface vpnInterface, String vpnName) {
2223         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2224                .get(vpnName);
2225         if (vpnInterfaces == null) {
2226             vpnInterfaces = new ConcurrentLinkedQueue<>();
2227         }
2228         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2229         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2230         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2231                 vpnInterface.getName(), vpnName);
2232     }
2233
2234     public boolean isVpnInstanceReady(String vpnInstanceName) {
2235         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2236         if (vpnRd == null) {
2237             return false;
2238         }
2239         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2240
2241         return vpnInstanceOpDataEntry != null;
2242     }
2243
2244     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2245         // FIXME: separate out to somehow?
2246         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2247         lock.lock();
2248         try {
2249             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2250                     unprocessedVpnInterfaces.get(vpnInstanceName);
2251             if (vpnInterfaces != null) {
2252                 while (!vpnInterfaces.isEmpty()) {
2253                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2254                     if (hasVpnInstanceCreatedSuccessfully) {
2255                         processSavedInterface(savedInterface, vpnInstanceName);
2256                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2257                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2258                     } else {
2259                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2260                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2261                     }
2262                 }
2263             } else {
2264                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2265             }
2266         } finally {
2267             lock.unlock();
2268         }
2269     }
2270
2271     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2272             VpnInterface vpnInterface) {
2273         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2274         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2275         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2276         lock.lock();
2277         try {
2278             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2279                     unprocessedVpnInterfaces.get(firstVpnName);
2280             if (vpnInterfaces != null) {
2281                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2282                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2283                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2284                 }
2285             } else {
2286                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2287             }
2288         } finally {
2289             lock.unlock();
2290         }
2291     }
2292
2293     public void vpnInstanceIsReady(String vpnInstanceName) {
2294         processSavedInterfaces(vpnInstanceName, true);
2295     }
2296
2297     public void vpnInstanceFailed(String vpnInstanceName) {
2298         processSavedInterfaces(vpnInstanceName, false);
2299     }
2300
2301     private static class UnprocessedVpnInterfaceData {
2302         InstanceIdentifier<VpnInterface> identifier;
2303         VpnInterface vpnInterface;
2304
2305         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2306             this.identifier = identifier;
2307             this.vpnInterface = vpnInterface;
2308         }
2309
2310         @Override
2311         public int hashCode() {
2312             final int prime = 31;
2313             int result = 1;
2314             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2315             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2316             return result;
2317         }
2318
2319         @Override
2320         public boolean equals(Object obj) {
2321             if (this == obj) {
2322                 return true;
2323             }
2324             if (obj == null) {
2325                 return false;
2326             }
2327             if (getClass() != obj.getClass()) {
2328                 return false;
2329             }
2330             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2331             if (identifier == null) {
2332                 if (other.identifier != null) {
2333                     return false;
2334                 }
2335             } else if (!identifier.equals(other.identifier)) {
2336                 return false;
2337             }
2338             if (vpnInterface == null) {
2339                 if (other.vpnInterface != null) {
2340                     return false;
2341                 }
2342             } else if (!vpnInterface.equals(other.vpnInterface)) {
2343                 return false;
2344             }
2345             return true;
2346         }
2347     }
2348
2349     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2350         String primaryRd = vpnUtil.getVpnRd(vpnName);
2351         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2352         if (vpnInstanceOpData == null) {
2353             return;
2354         }
2355         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2356         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2357             return;
2358         }
2359         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2360         vpnToDpnLists.forEach(vpnToDpnList -> {
2361             if (vpnToDpnList.getVpnInterfaces() == null) {
2362                 return;
2363             }
2364             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2365                 try {
2366                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2367                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2368                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2369                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2370                     if (!vpnInterfaceOptional.isPresent()) {
2371                         return;
2372                     }
2373                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2374                             vpnInterface.getInterfaceName());
2375                     if (configVpnAdjacencies == null) {
2376                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2377                         return;
2378                     }
2379                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2380                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2381                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2382                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2383                     // in operational DS. These unprocessed adjacencies will be handled below.
2384                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2385                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2386                     configVpnAdjacencies.stream()
2387                         .filter(adjacency -> operationVpnAdjacencies.stream()
2388                                 .noneMatch(operationalAdjacency ->
2389                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2390                         .forEach(adjacency -> {
2391                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2392                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2393                                 () -> {
2394                                     // TODO Deal with sequencing — the config tx must only submitted
2395                                     // if the oper tx goes in
2396                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2397                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2398                                         futures.add(
2399                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2400                                                 //set of prefix used, as entry in prefix-to-interface datastore
2401                                                 // is prerequisite for refresh Fib to avoid race condition leading
2402                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2403                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2404                                                 ListenableFuture<Void> configTxFuture =
2405                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2406                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2407                                                             primaryRd, adjacency,
2408                                                             vpnInterfaceOptional.get().getDpnId(),
2409                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2410                                                 Futures.addCallback(configTxFuture,
2411                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2412                                                     MoreExecutors.directExecutor());
2413                                                 futures.add(configTxFuture);
2414                                             }));
2415                                         return futures;
2416                                     } else {
2417                                         return emptyList();
2418                                     }
2419                                 });
2420                         });
2421                 } catch (ReadFailedException e) {
2422                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2423                             vpnName, primaryRd);
2424                 }
2425             });
2426         });
2427     }
2428
2429     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2430         private final String interfaceName;
2431         private final boolean add;
2432         private final String txnDestination;
2433
2434         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2435             this.interfaceName = interfaceName;
2436             this.add = add;
2437             this.txnDestination = transactionDest;
2438         }
2439
2440         @Override
2441         public void onSuccess(Void voidObj) {
2442             if (add) {
2443                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2444                         interfaceName, txnDestination);
2445             } else {
2446                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2447             }
2448         }
2449
2450         @Override
2451         public void onFailure(Throwable throwable) {
2452             if (add) {
2453                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2454                         interfaceName, txnDestination, throwable);
2455             } else {
2456                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2457                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2458             }
2459         }
2460     }
2461
2462     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2463         private final String primaryRd;
2464         private final Set<String> prefixListForRefreshFib;
2465
2466         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2467             this.primaryRd = primaryRd;
2468             this.prefixListForRefreshFib = prefixListForRefreshFib;
2469         }
2470
2471         @Override
2472         public void onSuccess(Void voidObj) {
2473             prefixListForRefreshFib.forEach(prefix -> {
2474                 fibManager.refreshVrfEntry(primaryRd, prefix);
2475             });
2476         }
2477
2478         @Override
2479         public void onFailure(Throwable throwable) {
2480             LOG.debug("write Tx config operation failed", throwable);
2481         }
2482     }
2483 }