vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.util.ArrayList;
  25 import java.util.Collections;
  26 import java.util.HashSet;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.locks.ReentrantLock;
  35 import java.util.function.Consumer;
  36 import java.util.function.Predicate;
  37 import java.util.stream.Collectors;
  38 import javax.annotation.PostConstruct;
  39 import javax.annotation.PreDestroy;
  40 import javax.inject.Inject;
  41 import javax.inject.Singleton;
  42 import org.eclipse.jdt.annotation.Nullable;
  43 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  46 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  47 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  48 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  49 import org.opendaylight.genius.infra.Datastore.Configuration;
  50 import org.opendaylight.genius.infra.Datastore.Operational;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  52 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  53 import org.opendaylight.genius.infra.TransactionAdapter;
  54 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  55 import org.opendaylight.genius.infra.TypedWriteTransaction;
  56 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  57 import org.opendaylight.genius.mdsalutil.NWUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.genius.utils.JvmGlobalLocks;
  62 import org.opendaylight.infrautils.caches.CacheProvider;
  63 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  64 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  65 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  66 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  67 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  68 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  69 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  70 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  71 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  72 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  73 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  74 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  75 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  79 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 @Singleton
 127 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 128
 129     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 130     private static final short DJC_MAX_RETRIES = 3;
 131
 132     private final DataBroker dataBroker;
 133     private final ManagedNewTransactionRunner txRunner;
 134     private final IBgpManager bgpManager;
 135     private final IFibManager fibManager;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IdManagerService idManager;
 138     private final OdlInterfaceRpcService ifaceMgrRpcService;
 139     private final VpnFootprintService vpnFootprintService;
 140     private final IInterfaceManager interfaceManager;
 141     private final IVpnManager vpnManager;
 142     private final ArpResponderHandler arpResponderHandler;
 143     private final JobCoordinator jobCoordinator;
 144     private final VpnUtil vpnUtil;
 145
 146     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 147
 148     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 149             new ConcurrentHashMap<>();
 150
 151     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 152
 153     @Inject
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final VpnFootprintService vpnFootprintService,
 161                                final IInterfaceManager interfaceManager,
 162                                final IVpnManager vpnManager,
 163                                final ArpResponderHandler arpResponderHandler,
 164                                final JobCoordinator jobCoordinator,
 165                                final CacheProvider cacheProvider,
 166                                final VpnUtil vpnUtil) {
 167         super(VpnInterface.class, VpnInterfaceManager.class);
 168
 169         this.dataBroker = dataBroker;
 170         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 171         this.bgpManager = bgpManager;
 172         this.idManager = idManager;
 173         this.mdsalManager = mdsalManager;
 174         this.fibManager = fibManager;
 175         this.ifaceMgrRpcService = ifaceMgrRpcService;
 176         this.vpnFootprintService = vpnFootprintService;
 177         this.interfaceManager = interfaceManager;
 178         this.vpnManager = vpnManager;
 179         this.arpResponderHandler = arpResponderHandler;
 180         this.jobCoordinator = jobCoordinator;
 181         this.vpnUtil = vpnUtil;
 182
 183         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 184                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 185                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 186     }
 187
 188     public Runnable isNotifyTaskQueued(String intfName) {
 189         return vpnIntfMap.remove(intfName);
 190     }
 191
 192     @PostConstruct
 193     public void start() {
 194         LOG.info("{} start", getClass().getSimpleName());
 195         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 196     }
 197
 198     @Override
 199     @PreDestroy
 200     public void close() {
 201         super.close();
 202         vpnInstanceOpDataEntryCache.close();
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 207         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 208     }
 209
 210     @Override
 211     protected VpnInterfaceManager getDataTreeChangeListener() {
 212         return VpnInterfaceManager.this;
 213     }
 214
 215     @Override
 216     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 217         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 218         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 219                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 220         addVpnInterface(identifier, vpnInterface, null, null);
 221     }
 222
 223     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 224                           final VpnInterface vpnInterface, String vpnName) {
 225         // FIXME: separate this out somehow?
 226         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 227         lock.lock();
 228         try {
 229             if (isVpnInstanceReady(vpnName)) {
 230                 return true;
 231             }
 232             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 233             return false;
 234         } finally {
 235             lock.unlock();
 236         }
 237     }
 238
 239     // TODO Clean up the exception handling
 240     @SuppressWarnings("checkstyle:IllegalCatch")
 241     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 242                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 243         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 244             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 245             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 246         }
 247     }
 248
 249     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 250                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 251         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 252         final String interfaceName = key.getName();
 253
 254         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 255             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 256                   vpnName, vpnInterface.getName());
 257             return;
 258         }
 259         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 260              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 261         List<Adjacency> copyOldAdjs = null;
 262         if (oldAdjs != null) {
 263             copyOldAdjs = new ArrayList<>();
 264             copyOldAdjs.addAll(oldAdjs);
 265         }
 266         List<Adjacency> copyNewAdjs = null;
 267         if (newAdjs != null) {
 268             copyNewAdjs = new ArrayList<>();
 269             copyNewAdjs.addAll(newAdjs);
 270         }
 271         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 272     }
 273
 274     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 275                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 276                     final @Nullable List<Adjacency> newAdjs,
 277                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 278         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 279         final String interfaceName = key.getName();
 280         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 281         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 282             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 283             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 284             if (interfaceState != null) {
 285                 try {
 286                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 287                     final int ifIndex = interfaceState.getIfIndex();
 288                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 289                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 290                         // (the inventory tx goes in last)
 291                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 292                         //set of prefix used, as entry in prefix-to-interface datastore
 293                         // is prerequisite for refresh Fib to avoid race condition leading to
 294                         // missing remote next hop in bucket actions on bgp-vpn delete
 295                         Set<String> prefixListForRefreshFib = new HashSet<>();
 296                         ListenableFuture<Void> confFuture =
 297                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 298                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 299                                     operTx -> futures.add(
 300                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 301                                             LOG.info(
 302                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 303                                                     + " on dpn {}",
 304                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 305                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 306                                                 confTx, operTx, invTx, interfaceState, vpnName,
 307                                                 prefixListForRefreshFib);
 308                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 309                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 310                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 311                                                 if (newAdjs != null) {
 312                                                     for (Adjacency adj : newAdjs) {
 313                                                         if (oldAdjs.contains(adj)) {
 314                                                             oldAdjs.remove(adj);
 315                                                         } else {
 316                                                             if (!isBgpVpnInternetVpn
 317                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 318                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 319                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 320                                                                     prefixListForRefreshFib);
 321                                                             }
 322                                                         }
 323                                                     }
 324                                                 }
 325                                                 for (Adjacency adj : oldAdjs) {
 326                                                     if (!isBgpVpnInternetVpn
 327                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 328                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 329                                                             operTx, confTx);
 330                                                     }
 331                                                 }
 332                                             }
 333                                         })))));
 334                         Futures.addCallback(confFuture,
 335                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 336                             MoreExecutors.directExecutor());
 337                         futures.add(confFuture);
 338                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 339                             MoreExecutors.directExecutor());
 340                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 341                             + " processed successfully", interfaceName, vpnName, dpnId);
 342                         return futures;
 343                     });
 344                 } catch (NumberFormatException | IllegalStateException e) {
 345                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 346                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 347                         vpnName, e);
 348                     return;
 349                 }
 350             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 351                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 352                     () -> {
 353                         ListenableFuture<Void> future =
 354                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 355                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 356                                     confTx, vpnName);
 357                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 358                                     vpnName, vpnInterface.getDpnId());
 359                             });
 360                         ListenableFutures.addErrorLogging(future, LOG,
 361                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 362                         return Collections.singletonList(future);
 363                     });
 364             } else {
 365                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 366                     + " is not available", interfaceName, vpnName);
 367             }
 368         } else {
 369             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 370                     + " as vpn is pending delete", interfaceName, vpnName,
 371                 vpnInterface.getDpnId());
 372         }
 373     }
 374
 375     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 376     // see comments below.
 377     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 378     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 379             final int lportTag, boolean isInterfaceUp,
 380             TypedWriteTransaction<Configuration> writeConfigTxn,
 381             TypedWriteTransaction<Operational> writeOperTxn,
 382             TypedReadWriteTransaction<Configuration> writeInvTxn,
 383             Interface interfaceState, final String vpnName,
 384             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 385         final String interfaceName = vpnInterface.getName();
 386         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 387                 vpnName);
 388         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 389         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 390         if (!isInterfaceUp) {
 391             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 392                      interfaceName, dpId, vpnName);
 393             long vpnId = vpnUtil.getVpnId(vpnName);
 394             if (vpnId == VpnConstants.INVALID_ID) {
 395                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 396                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 397                         dpId);
 398                 return;
 399             }
 400
 401             boolean waitForVpnInterfaceOpRemoval = false;
 402             if (opVpnInterface != null) {
 403                 String opVpnName = opVpnInterface.getVpnInstanceName();
 404                 String primaryInterfaceIp = null;
 405                 if (Objects.equals(opVpnName, vpnName)) {
 406                     // Please check if the primary VRF Entry does not exist for VPNInterface
 407                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 408                     // back to back
 409                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 410                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 411                     if (adjs == null) {
 412                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 413                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 414                                 vpnName);
 415                         return;
 416                     }
 417                     for (Adjacency adj : adjs) {
 418                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 419                             primaryInterfaceIp = adj.getIpAddress();
 420                             break;
 421                         }
 422                     }
 423                     if (primaryInterfaceIp == null) {
 424                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 425                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 426                                 dpId, vpnName);
 427                         return;
 428                     }
 429                     // Get the rd of the vpn instance
 430                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 431                     if (vrf != null) {
 432                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 433                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 434                         return;
 435                     }
 436                     waitForVpnInterfaceOpRemoval = true;
 437                 } else {
 438                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 439                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 440                 }
 441             }
 442             if (!waitForVpnInterfaceOpRemoval) {
 443                 // Add the VPNInterface and quit
 444                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 445                         null/*ipAddressSourceValuePair*/,
 446                         true /* add */);
 447                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 448                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 449                 if (!isBgpVpnInternetVpn) {
 450                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 451                 }
 452                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 453                         dpId, vpnName);
 454                 if (interfaceManager.isExternalInterface(interfaceName)) {
 455                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 456                         NwConstants.ADD_FLOW);
 457                 }
 458                 return;
 459             }
 460
 461             // FIB didn't get a chance yet to clean up this VPNInterface
 462             // Let us give it a chance here !
 463             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 464                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 465             try {
 466                 Runnable notifyTask = new VpnNotifyTask();
 467                 synchronized (notifyTask) {
 468                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 469                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 470                     // published here while holding the lock on it so this path will hit the wait before notify can be
 471                     // invoked.
 472                     vpnIntfMap.put(interfaceName, notifyTask);
 473                     try {
 474                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 475                     } catch (InterruptedException e) {
 476                         // Ignored
 477                     }
 478                 }
 479             } finally {
 480                 vpnIntfMap.remove(interfaceName);
 481             }
 482
 483             if (opVpnInterface != null) {
 484                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 485                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 486                         dpId, vpnName);
 487                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 488                 return;
 489             }
 490             // VPNInterface got removed, proceed with Add
 491             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 492                     interfaceName, dpId, vpnName);
 493             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 494                     null/*ipAddressSourceValuePair*/,
 495                     true /* add */);
 496             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 497                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 498             if (!isBgpVpnInternetVpn) {
 499                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 500             }
 501             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 502                     + " FIB to clean up", interfaceName, dpId, vpnName);
 503             if (interfaceManager.isExternalInterface(interfaceName)) {
 504                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 505                                lportTag, NwConstants.ADD_FLOW);
 506             }
 507         } else {
 508             try {
 509                 // Interface is retained in the DPN, but its Link Up.
 510                 // Advertise prefixes again for this interface to BGP
 511                 InstanceIdentifier<VpnInterface> identifier =
 512                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 513                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 514                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 515                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 516                 // Perform similar operation as interface add event for extraroutes.
 517                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 518                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 519                         LogicalDatastoreType.CONFIGURATION, path);
 520                 if (!optAdjacencies.isPresent()) {
 521                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 522                     return;
 523                 }
 524                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 525                 for (Adjacency adjacency : adjacencies) {
 526                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 527                         continue;
 528                     }
 529                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 530                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 531                         continue;
 532                     }
 533                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 534                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 535                 }
 536             } catch (ReadFailedException e) {
 537                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 538                         interfaceName, vpnName, primaryRd, dpId);
 539             }
 540         }
 541     }
 542
 543     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 544         int lportTag, int addOrRemove) {
 545         Uuid extNetworkId;
 546         try {
 547             // vpn instance of ext-net interface is the network-id
 548             extNetworkId = new Uuid(vpnName);
 549         } catch (IllegalArgumentException e) {
 550             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 551                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 552             return;
 553         }
 554
 555         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 556         if (routerIds == null || routerIds.isEmpty()) {
 557             LOG.info("processExternalVpnInterface: No router is associated with {}."
 558                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 559                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 560             return;
 561         }
 562
 563         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 564                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 565         for (Uuid routerId : routerIds) {
 566             String routerName = routerId.getValue();
 567             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 568             if (Objects.equals(primarySwitch, dpId)) {
 569                 Routers router = vpnUtil.getExternalRouter(routerName);
 570                 if (router != null) {
 571                     if (addOrRemove == NwConstants.ADD_FLOW) {
 572                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 573                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 574                                 dpId, interfaceName, lportTag);
 575                     } else {
 576                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 577                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 578                                 dpId, interfaceName, lportTag);
 579                     }
 580                 } else {
 581                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 582                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 583                             interfaceName, dpId, vpnName);
 584                 }
 585             }
 586         }
 587     }
 588
 589     // TODO Clean up the exception handling
 590     @SuppressWarnings("checkstyle:IllegalCatch")
 591     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 592                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 593                                                   String vpnName, String interfaceName) {
 594         if (rd == null) {
 595             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 596                   interfaceName, dpnId, vpnName);
 597             return;
 598         } else {
 599             if (rd.equals(vpnName)) {
 600                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 601                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 602                 return;
 603             }
 604         }
 605         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 606                 interfaceName, dpnId, vpnName, rd);
 607
 608         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 609         if (nextHopIp == null) {
 610             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 611                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 612                     rd, vpnName);
 613             return;
 614         }
 615
 616         try {
 617             //Read NextHops
 618             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 619             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 620                     LogicalDatastoreType.OPERATIONAL, path);
 621             if (adjacencies.isPresent()) {
 622                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 623                 if (nextHops != null && !nextHops.isEmpty()) {
 624                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 625                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 626                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 627                     long l3vni = vpnInstanceOpData.getL3vni();
 628                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 629                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 630                     for (Adjacency nextHop : nextHops) {
 631                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 632                             continue;
 633                         }
 634                         String gatewayMac = null;
 635                         long label = 0;
 636                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 637                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 638                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 639                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 640                                     .get();
 641                         } else {
 642                             label = nextHop.getLabel();
 643                         }
 644                         try {
 645                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 646                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 647                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 648                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 649                                     gatewayMac);
 650                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 651                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 652                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 653                         } catch (Exception e) {
 654                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 655                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 656                                     interfaceName, dpnId, e);
 657                         }
 658                     }
 659                 }
 660             }
 661         } catch (ReadFailedException e) {
 662             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 663                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 664         }
 665     }
 666
 667     // TODO Clean up the exception handling
 668     @SuppressWarnings("checkstyle:IllegalCatch")
 669     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 670                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 671                         TypedWriteTransaction<Operational> writeOperTx) {
 672         //Read NextHops
 673         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 674         String rd = vpnUtil.getVpnRd(interfaceName);
 675         if (rd == null) {
 676             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 677                 interfaceName, vpnName);
 678             return;
 679         } else {
 680             if (rd.equals(vpnName)) {
 681                 LOG.info(
 682                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 683                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 684                 return;
 685             }
 686         }
 687         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 688                vpnName, rd);
 689         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 690         try {
 691             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 692                     path);
 693         } catch (ReadFailedException e) {
 694             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 695                     interfaceName, vpnName);
 696         }
 697         if (adjacencies.isPresent()) {
 698             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 699
 700             if (nextHops != null && !nextHops.isEmpty()) {
 701                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 702                         nextHops, interfaceName, vpnName, rd);
 703                 for (Adjacency nextHop : nextHops) {
 704                     try {
 705                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 706                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 707                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 708                                     interfaceName, vpnName);
 709                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 710                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 711                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 712                                     interfaceName, vpnName);
 713                         } else if (nextHop.getNextHopIpList() != null) {
 714                             // Perform similar operation as interface delete event for extraroutes.
 715                             String allocatedRd = nextHop.getVrfId();
 716                             for (String nh : nextHop.getNextHopIpList()) {
 717                                 deleteExtraRouteFromCurrentAndImportingVpns(
 718                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 719                                     writeOperTx);
 720                             }
 721                         }
 722                     } catch (Exception e) {
 723                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 724                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 725                     }
 726                 }
 727             }
 728         }
 729     }
 730
 731     @SuppressWarnings("checkstyle:IllegalCatch")
 732     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 733                                                   String primaryRd, String interfaceName, final long vpnId,
 734                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 735                                                   TypedWriteTransaction<Operational> writeOperTxn,
 736                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 737                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 738             throws ExecutionException, InterruptedException {
 739         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 740         // Read NextHops
 741         Optional<VpnInterface> vpnInteface = Optional.absent();
 742         try {
 743             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 744             LogicalDatastoreType.CONFIGURATION, identifier);
 745         } catch (ReadFailedException e) {
 746             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 747                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 748         }
 749         Uuid intfnetworkUuid = null;
 750         NetworkType networkType = null;
 751         Long segmentationId = Long.valueOf(-1);
 752         Adjacencies adjacencies = null;
 753         if (vpnInteface.isPresent()) {
 754             intfnetworkUuid = vpnInteface.get().getNetworkId();
 755             networkType = vpnInteface.get().getNetworkType();
 756             segmentationId = vpnInteface.get().getSegmentationId();
 757             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 758             if (adjacencies == null) {
 759                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 760                         null/*gwMac*/, writeOperTxn);
 761                 return;
 762             }
 763         }
 764         // Get the rd of the vpn instance
 765         String nextHopIp = null;
 766         try {
 767             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 768         } catch (Exception e) {
 769             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 770                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 771         }
 772         List<String> nhList = new ArrayList<>();
 773         if (nextHopIp != null) {
 774             nhList.add(nextHopIp);
 775             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 776                     interfaceName, dpnId, vpnName, nhList);
 777         }
 778         Optional<String> gwMac = Optional.absent();
 779         String vpnInterfaceSubnetGwMacAddress = null;
 780         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 781         Long l3vni = vpnInstanceOpData.getL3vni();
 782         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 783         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 784         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 785         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 786         List<Adjacency> value = new ArrayList<>();
 787         for (Adjacency nextHop : nextHops) {
 788             String rd = primaryRd;
 789             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 790             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 791                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 792                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 793                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 794                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 795                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 796                 continue;
 797             }
 798             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 799                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 800                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 801                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 802                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 803                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 804
 805                 Prefixes prefixes = intfnetworkUuid != null
 806                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 807                             segmentationId, prefixCue) :
 808                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 809                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 810                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 811                 final Uuid subnetId = nextHop.getSubnetId();
 812
 813                 String gatewayIp = nextHop.getSubnetGatewayIp();
 814                 if (gatewayIp == null) {
 815                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 816                     if (gatewayIpOptional.isPresent()) {
 817                         gatewayIp = gatewayIpOptional.get();
 818                     }
 819                 }
 820
 821                 if (gatewayIp != null) {
 822                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 823                     if (gwMac.isPresent()) {
 824                         // A valid mac-address is available for this subnet-gateway-ip
 825                         // Use this for programming ARP_RESPONDER table here.  And save this
 826                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 827                         // to populate L3_GW_MAC_TABLE there.
 828                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 829                                 gatewayIp, gwMac.get());
 830                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 831                     } else {
 832                         // A valid mac-address is not available for this subnet-gateway-ip
 833                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 834                         // Save this connected-mac-address as gateway-mac-address for the
 835                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 836                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 837                         if (gwMac.isPresent()) {
 838                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 839                                     NwConstants.ADD_FLOW, gwMac.get());
 840                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 841                                     gatewayIp, gwMac.get());
 842                         } else {
 843                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 844                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 845                                 + "gwIp {}",
 846                                 subnetId, interfaceName, vpnName, gatewayIp);
 847                         }
 848                     }
 849                 } else {
 850                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 851                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 852                         subnetId, interfaceName, vpnName);
 853                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 854                 }
 855                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 856                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 857             } else {
 858                 //Extra route adjacency
 859                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 860                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 861                 // FIXME: separate this out somehow?
 862                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 863                 lock.lock();
 864                 try {
 865                     java.util.Optional<String> rdToAllocate = vpnUtil
 866                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 867                                     nextHop.getNextHopIpList().get(0), dpnId);
 868                     if (rdToAllocate.isPresent()) {
 869                         rd = rdToAllocate.get();
 870                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 871                             rd, prefix);
 872                     } else {
 873                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 874                         continue;
 875                     }
 876                 } finally {
 877                     lock.unlock();
 878                 }
 879                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 880                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 881                         interfaceName, dpnId);
 882             }
 883             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 884             // can be different from the gateway-mac-address within the VRFEntry as the
 885             // gateway-mac-address is a superset.
 886             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 887             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 888                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 889                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 890             Adjacency operationalAdjacency = null;
 891             try {
 892                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 893             } catch (NullPointerException e) {
 894                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 895                     input, e.getMessage());
 896                 return;
 897             }
 898             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 899                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 900                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 901                     writeConfigTxn);
 902             }
 903             value.add(operationalAdjacency);
 904         }
 905
 906         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 907         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 908                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 909
 910         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 911                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 912                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 913
 914         for (Adjacency nextHop : aug.getAdjacency()) {
 915             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 916             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 917                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 918                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 919                 registeredPopulator.populateFib(input, writeConfigTxn);
 920             }
 921         }
 922     }
 923
 924     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 925                                               long lportTag, String gwMac,
 926                                               TypedWriteTransaction<Operational> writeOperTxn) {
 927         VpnInterfaceOpDataEntry opInterface =
 928               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 929         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 930             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 931         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 932         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 933                 interfaceName, dpnId, vpnName);
 934     }
 935
 936     // TODO Clean up the exception handling
 937     @SuppressWarnings("checkstyle:IllegalCatch")
 938     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 939                                            StateTunnelList stateTunnelList,
 940                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 941                                            TypedWriteTransaction<Operational> writeOperTxn) {
 942
 943         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 944         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 945         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 946         List<Adjacency> adjList =
 947             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 948         if (adjList.isEmpty()) {
 949             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 950                     vpnInterface, srcDpnId);
 951             return;
 952         }
 953         String prefix = null;
 954         long label = 0;
 955         List<Adjacency> value = new ArrayList<>();
 956         boolean isNextHopAddReqd = false;
 957         String vpnName = vpnInterface.getVpnInstanceName();
 958         long vpnId = vpnUtil.getVpnId(vpnName);
 959         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 960         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 961                 vpnInterface.getName(), vpnInterface.getDpnId(),
 962                 vpnInterface.getVpnInstanceName(), adjList);
 963         for (Adjacency adj : adjList) {
 964             String rd = adj.getVrfId();
 965             rd = rd != null ? rd : vpnName;
 966             prefix = adj.getIpAddress();
 967             label = adj.getLabel();
 968             List<String> nhList = Collections.singletonList(srcTepIp);
 969             List<String> nextHopList = adj.getNextHopIpList();
 970             // If TEP is added , update the nexthop of primary adjacency.
 971             // Secondary adj nexthop is already pointing to primary adj IP address.
 972             if (nextHopList == null || nextHopList.isEmpty()) {
 973                 isNextHopAddReqd = true;
 974             }
 975
 976             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 977                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 978             } else {
 979                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 980                 if (!vrfEntryOptional.isPresent()) {
 981                     continue;
 982                 }
 983                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 984                 if (!nhList.contains(srcTepIp)) {
 985                     nhList.add(srcTepIp);
 986                     isNextHopAddReqd = true;
 987                 }
 988                 value.add(adj);
 989             }
 990
 991             if (isNextHopAddReqd) {
 992                 updateLabelMapper(label, nhList);
 993                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 994                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 995                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 996                 // Update the VRF entry with nextHop
 997                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 998                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 999
1000                 //Get the list of VPN's importing this route(prefix) .
1001                 // Then update the VRF entry with nhList
1002                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1003                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1004                     String vpnRd = vpn.getVrfId();
1005                     if (vpnRd != null) {
1006                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1007                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1008                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1009                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1010                             vpnInterface.getName(), srcDpnId, vpnName,
1011                             vpn.getVpnInstanceName(), vpnRd);
1012                     }
1013                 }
1014                 // Advertise the prefix to BGP only for external vpn
1015                 // since there is a nexthop change.
1016                 try {
1017                     if (!rd.equalsIgnoreCase(vpnName)) {
1018                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1019                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1020                                 null /*gatewayMacAddress*/);
1021                     }
1022                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1023                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1024                             srcDpnId, vpnName);
1025                 } catch (Exception ex) {
1026                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1027                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1028                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1029                 }
1030             }
1031         }
1032         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1033         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1034                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1035                 .addAugmentation(AdjacenciesOp.class, aug).build();
1036         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1037                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1038         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1039         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1040                 vpnInterface.getName(), srcDpnId, vpnName);
1041
1042     }
1043
1044     // TODO Clean up the exception handling
1045     @SuppressWarnings("checkstyle:IllegalCatch")
1046     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1047                                               StateTunnelList stateTunnelList,
1048                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1049                                               TypedWriteTransaction<Operational> writeOperTxn) {
1050
1051         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1052         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1053         String prefix = null;
1054         long label = 0;
1055         boolean isNextHopRemoveReqd = false;
1056         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1057         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1058         String vpnName = vpnInterface.getVpnInstanceName();
1059         long vpnId = vpnUtil.getVpnId(vpnName);
1060         String primaryRd = vpnUtil.getVpnRd(vpnName);
1061         if (adjList != null) {
1062             List<Adjacency> value = new ArrayList<>();
1063             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1064                     vpnInterface.getName(), vpnInterface.getDpnId(),
1065                     vpnInterface.getVpnInstanceName(), adjList);
1066             for (Adjacency adj : adjList) {
1067                 List<String> nhList = new ArrayList<>();
1068                 String rd = adj.getVrfId();
1069                 rd = rd != null ? rd : vpnName;
1070                 prefix = adj.getIpAddress();
1071                 List<String> nextHopList = adj.getNextHopIpList();
1072                 label = adj.getLabel();
1073                 if (nextHopList != null && !nextHopList.isEmpty()) {
1074                     isNextHopRemoveReqd = true;
1075                 }
1076                 // If TEP is deleted , remove the nexthop from primary adjacency.
1077                 // Secondary adj nexthop will continue to point to primary adj IP address.
1078                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1079                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1080                 } else {
1081                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1082                     if (!vrfEntryOptional.isPresent()) {
1083                         continue;
1084                     }
1085                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1086                     if (nhList.contains(srcTepIp)) {
1087                         nhList.remove(srcTepIp);
1088                         isNextHopRemoveReqd = true;
1089                     }
1090                     value.add(adj);
1091                 }
1092
1093                 if (isNextHopRemoveReqd) {
1094                     updateLabelMapper(label, nhList);
1095                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1096                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1097                             prefix, nhList, vpnName,
1098                             vpnId, rd, vpnInterface.getName());
1099                     // Update the VRF entry with removed nextHop
1100                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1101                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1102
1103                     //Get the list of VPN's importing this route(prefix) .
1104                     // Then update the VRF entry with nhList
1105                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1106                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1107                         String vpnRd = vpn.getVrfId();
1108                         if (vpnRd != null) {
1109                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1110                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1111                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1112                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1113                                     nhList, label, vpnInterface.getName(), srcDpnId,
1114                                     vpnName,
1115                                     vpn.getVpnInstanceName(), vpnRd);
1116                         }
1117                     }
1118
1119                     // Withdraw prefix from BGP only for external vpn.
1120                     try {
1121                         if (!rd.equalsIgnoreCase(vpnName)) {
1122                             bgpManager.withdrawPrefix(rd, prefix);
1123                         }
1124                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1125                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1126                                 vpnInterface.getName(), srcDpnId,
1127                                 vpnName);
1128                     } catch (Exception ex) {
1129                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1130                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1131                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1132                     }
1133                 }
1134             }
1135             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1136             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1137                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1138                     .addAugmentation(AdjacenciesOp.class, aug).build();
1139             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1140                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1141             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1142             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1143                          vpnInterface.getName(), srcDpnId, vpnName);
1144         }
1145     }
1146
1147     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1148         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1149
1150         String vpnRd = vpnUtil.getVpnRd(vpnName);
1151         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1152         if (vpnInstanceOpDataEntry == null) {
1153             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1154                     + " to check for vpns exporting the routes", vpnName);
1155             return vpnsToExportRoute;
1156         }
1157
1158         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1159             if (input.getVpnInstanceName() == null) {
1160                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1161                         input.getVrfId());
1162                 return false;
1163             }
1164             return !input.getVpnInstanceName().equals(vpnName);
1165         };
1166
1167         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1168             Iterable<String> commonRTs =
1169                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1170                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1171             return Iterators.size(commonRTs.iterator()) > 0;
1172         };
1173
1174         vpnsToExportRoute =
1175                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1176                         Collectors.toList());
1177         return vpnsToExportRoute;
1178     }
1179
1180     // TODO Clean up the exception handling
1181     @SuppressWarnings("checkstyle:IllegalCatch")
1182     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1183         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1184         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1185             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1186             if (vrfEntries != null) {
1187                 ListenableFutures.addErrorLogging(
1188                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1189                         for (VrfEntry vrfEntry : vrfEntries) {
1190                             try {
1191                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1192                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1193                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1194                                             + " is not a controller managed non intervpn link route. Ignoring.",
1195                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1196                                     continue;
1197                                 }
1198                                 String prefix = vrfEntry.getDestPrefix();
1199                                 String gwMac = vrfEntry.getGatewayMacAddress();
1200                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1201                                     String nh = routePath.getNexthopAddress();
1202                                     int label = routePath.getLabel().intValue();
1203                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1204                                         vrfEntry.getOrigin()))) {
1205                                         LOG.info(
1206                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1207                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1208                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1209                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1210                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1211                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1212                                             confTx);
1213                                     } else {
1214                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1215                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1216                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1217                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1218                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1219                                             confTx);
1220                                     }
1221                                 });
1222                             } catch (RuntimeException e) {
1223                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1224                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1225                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1226                             }
1227                         }
1228                     }), LOG, "Error handing VPN exporting routes");
1229             } else {
1230                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1231                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1232             }
1233         }
1234     }
1235
1236     @Override
1237     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1238         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1239         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1240         final String interfaceName = key.getName();
1241         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1242             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1243             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1244         }
1245     }
1246
1247     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1248                                 final VpnInterface vpnInterface, final String vpnName,
1249                                 final String interfaceName) {
1250         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1251             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1252                 ListenableFuture<Void> future =
1253                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1254                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1255                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1256                     });
1257                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1258                         vpnInterface.getName(), vpnName);
1259                 return Collections.singletonList(future);
1260             }, DJC_MAX_RETRIES);
1261         } else {
1262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1263             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1264         }
1265     }
1266
1267     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1268     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1269                                            final VpnInterface vpnInterface, final String vpnName,
1270                                            final String interfaceName, final Interface interfaceState) {
1271         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1272                 vpnName, vpnInterface.getDpnId());
1273         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1274         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1275             () -> {
1276                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1277                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1278                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1279                         writeOperTxn -> futures.add(
1280                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1281                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1282                                         interfaceName, vpnName);
1283                                 BigInteger dpId;
1284                                 int ifIndex;
1285                                 String gwMacAddress;
1286                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1287                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1288                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1289                                 try {
1290                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1291                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1292                                 } catch (ReadFailedException e) {
1293                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1294                                             interfaceName, vpnName);
1295                                     return;
1296                                 }
1297                                 if (interfaceState != null) {
1298                                     try {
1299                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1300                                     } catch (NumberFormatException | IllegalStateException e) {
1301                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1302                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1303                                                         + " from vpn interface op data store. ", interfaceName,
1304                                                 vpnInterface.getDpnId(), vpnName, e);
1305                                         dpId = BigInteger.ZERO;
1306                                     }
1307                                     ifIndex = interfaceState.getIfIndex();
1308                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1309                                 } else {
1310                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1311                                             + " from vpn interface op.", interfaceName);
1312                                     if (optVpnInterface.isPresent()) {
1313                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1314                                         dpId = vpnOpInterface.getDpnId();
1315                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1316                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1317                                     } else {
1318                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1319                                                 + " as interfaceState and vpn interface op is not"
1320                                                 + " available", interfaceName, vpnName);
1321                                         return;
1322                                     }
1323                                 }
1324                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1325                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1326                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1327                                 LOG.info(
1328                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1329                                                 + "successfully",
1330                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1331                             })))));
1332                 futures.add(configFuture);
1333                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1334                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1335                 return futures;
1336             }, DJC_MAX_RETRIES);
1337     }
1338
1339     protected void processVpnInterfaceDown(BigInteger dpId,
1340                                            String interfaceName,
1341                                            int lportTag,
1342                                            String gwMac,
1343                                            VpnInterfaceOpDataEntry vpnOpInterface,
1344                                            boolean isInterfaceStateDown,
1345                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1346                                            TypedWriteTransaction<Operational> writeOperTxn,
1347                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1348             throws ExecutionException, InterruptedException {
1349         if (vpnOpInterface == null) {
1350             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1351                     + " as it is not available in operational data store", interfaceName, dpId);
1352             return;
1353         }
1354         final String vpnName = vpnOpInterface.getVpnInstanceName();
1355         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1356                                                     interfaceName, vpnName);
1357         if (!isInterfaceStateDown) {
1358             final long vpnId = vpnUtil.getVpnId(vpnName);
1359             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1360             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1361             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1362                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1363             if (interfaceManager.isExternalInterface(interfaceName)) {
1364                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1365                     NwConstants.DEL_FLOW);
1366             }
1367             if (!isBgpVpnInternetVpn) {
1368                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1369             }
1370             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1371                     + " successful", interfaceName, dpId, vpnName);
1372         } else {
1373             // Interface is retained in the DPN, but its Link Down.
1374             // Only withdraw the prefixes for this interface from BGP
1375             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1376         }
1377     }
1378
1379     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1380                                           final String vpnName, final long vpnId, String gwMac,
1381                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1382                                           TypedWriteTransaction<Operational> writeOperTxn,
1383                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1384             throws ExecutionException, InterruptedException {
1385         //Read NextHops
1386         try {
1387             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1388                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1389             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1390             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1391                     LogicalDatastoreType.OPERATIONAL, path);
1392             boolean isNonPrimaryAdjIp = Boolean.FALSE;
1393             String primaryRd = vpnUtil.getVpnRd(vpnName);
1394             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1395                     interfaceName, dpnId, vpnName, primaryRd);
1396             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1397                     && !adjacencies.get().getAdjacency().isEmpty()) {
1398                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1399                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1400                         interfaceName, dpnId, vpnName, nextHops);
1401                 for (Adjacency nextHop : nextHops) {
1402                     if (nextHop.isPhysNetworkFunc()) {
1403                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1404                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1405                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1406                                 null/*writeCfgTxn*/);
1407                     } else {
1408                         String rd = nextHop.getVrfId();
1409                         List<String> nhList;
1410                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1411                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1412                             isNonPrimaryAdjIp = Boolean.TRUE;
1413                         } else {
1414                             // This is a primary adjacency
1415                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1416                                     : emptyList();
1417                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1418                                     interfaceName, writeInvTxn);
1419                         }
1420                         if (!nhList.isEmpty()) {
1421                             if (Objects.equals(primaryRd, vpnName)) {
1422                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1423                                 //remove from FIB directly
1424                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1425                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1426                             } else {
1427                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1428                                         interfaceName, writeConfigTxn, writeOperTxn);
1429                             }
1430                         } else {
1431                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1432                                             + " interface {}", nextHop.getIpAddress(), rd,
1433                                     nextHop.getAdjacencyType().toString(), interfaceName);
1434                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1435                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1436                         }
1437                     }
1438                     String ip = nextHop.getIpAddress().split("/")[0];
1439                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1440                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1441                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1442                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1443                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1444                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1445                     }
1446                     // Remove the MIP-IP from VpnPortIpToPort.
1447                     if (isNonPrimaryAdjIp) {
1448                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1449                         if (persistedIp != null && persistedIp.isLearntIp()
1450                                 && persistedIp.getPortName().equals(interfaceName)) {
1451                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1452                             LOG.info(
1453                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1454                                             + "from VpnPortipToPort",
1455                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1456                         }
1457                     }
1458                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1459                     if (vpnPortipToPort != null) {
1460                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1461                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1462                                  + "Interface {} ip {} on dpn {} for vpn {}",
1463                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1464                     }
1465                 }
1466             } else {
1467                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1468                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1469                         + " Removing it.", interfaceName, vpnName, dpnId);
1470                 writeOperTxn.delete(identifier);
1471             }
1472         } catch (ReadFailedException e) {
1473             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1474                     interfaceName, dpnId, vpnName);
1475         }
1476     }
1477
1478     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1479                                                             String interfaceName, BigInteger dpnId,
1480                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1481                                                             TypedWriteTransaction<Operational> writeOperTx) {
1482         return (nh) -> {
1483             String primaryRd = vpnUtil.getVpnRd(vpnName);
1484             String prefix = nextHop.getIpAddress();
1485             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1486             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1487                     nextHop, vpnName, interfaceName, dpnId);
1488             // FIXME: separate this out somehow?
1489             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1490             lock.lock();
1491             try {
1492                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1493                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1494                     //If extra-route is present behind at least one VM, then do not remove or update
1495                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1496                     //prefixtointerface DS
1497                     return;
1498                 }
1499                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1500                         writeConfigTxn);
1501             } finally {
1502                 lock.unlock();
1503             }
1504             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1505                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1506                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1507         };
1508     }
1509
1510     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1511                                            BigInteger dpnId, String rd, String interfaceName,
1512                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1513                                            TypedWriteTransaction<Operational> writeOperTx) {
1514         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1515                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1516         nhList.forEach((nh) -> {
1517             //IRT: remove routes from other vpns importing it
1518             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1519                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1520             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1521                 String vpnRd = vpn.getVrfId();
1522                 if (vpnRd != null) {
1523                     fibManager.removeOrUpdateFibEntry(vpnRd,
1524                             nextHop.getIpAddress(), nh, writeConfigTxn);
1525                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1526                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1527                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1528                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1529                 }
1530             }
1531         });
1532     }
1533
1534     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1535                                                  int lportTag, String gwMac, String interfaceName,
1536                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1537             throws ExecutionException, InterruptedException {
1538         final Uuid subnetId = nextHop.getSubnetId();
1539         if (nextHop.getSubnetGatewayMacAddress() == null) {
1540             // A valid mac-address was not available for this subnet-gateway-ip
1541             // So a connected-mac-address was used for this subnet and we need
1542             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1543             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1544         }
1545         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1546                 subnetId);
1547     }
1548
1549     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1550                                                   String interfaceName) {
1551         // This is either an extra-route (or) a learned IP via subnet-route
1552         List<String> nhList = null;
1553         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1554         if (nextHopIp == null || nextHopIp.isEmpty()) {
1555             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1556                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1557                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1558                     vpnName);
1559             nhList = emptyList();
1560         } else {
1561             nhList = Collections.singletonList(nextHopIp);
1562         }
1563         return nhList;
1564     }
1565
1566     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1567         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1568         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1569         // else use connected interface
1570         if (gwPort != null && gwPort.isSubnetIp()) {
1571             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1572                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1573             return Optional.of(gwPort.getMacAddress());
1574         }
1575         return Optional.absent();
1576     }
1577
1578     @Override
1579     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1580         final VpnInterface update) {
1581         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1582         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1583                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1584         final String vpnInterfaceName = update.getName();
1585         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1586         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1587         //handles switching between <internal VPN - external VPN>
1588         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1589             List<ListenableFuture<Void>> futures = new ArrayList<>();
1590             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1591                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1592                                 + "to newVpn(s) {}",
1593                         original.getName(), dpnId,
1594                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1595                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1596                 return emptyList();
1597             }
1598             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1599             return futures;
1600         });
1601     }
1602
1603     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1604                                                            VpnInterface original, VpnInterface update,
1605                                                            List<ListenableFuture<Void>> futures) {
1606         boolean isVpnInstanceUpdate = false;
1607         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1608         final String interfaceName = key.getName();
1609         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1610         List<String> oldVpnListCopy = new ArrayList<>();
1611         oldVpnListCopy.addAll(oldVpnList);
1612         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1613         List<String> newVpnListCopy = new ArrayList<>();
1614         newVpnListCopy.addAll(newVpnList);
1615
1616         oldVpnList.removeAll(newVpnList);
1617         newVpnList.removeAll(oldVpnListCopy);
1618         //This block will execute only on if there is a change in the VPN Instance.
1619         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1620             /*
1621              * Internet BGP-VPN Instance update with single router:
1622              * ====================================================
1623              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1624              *     1st VPN Instance : router VPN or external BGP-VPN.
1625              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1626              *
1627              * VPN Instance UPDATE:
1628              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1629              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1630              *
1631              * External BGP-VPN Instance update with single router:
1632              * ====================================================
1633              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1634              *
1635              * Updated VPN Instance will be always either internal router VPN to
1636              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1637              *
1638              * VPN Instance UPDATE:
1639              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1640              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1641              *
1642              * Dual Router VPN Instance Update:
1643              * ================================
1644              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1645              *
1646              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1647              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1648              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1649              *
1650              * Dual Router --> Associated with common external BGP-VPN Instance.
1651              * 1st router and 2nd router are getting associated with single External BGP-VPN
1652              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1653              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1654              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1655              *
1656              *
1657              */
1658             isVpnInstanceUpdate = true;
1659             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1660                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1661                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1662                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1663                     List<String> externalBgpVpnList = new ArrayList<>();
1664                     for (String newVpnName : newVpnListCopy) {
1665                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1666                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1667                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1668                                 .BgpvpnType.BGPVPNExternal) {
1669                             externalBgpVpnList.add(newVpnName);
1670                             break;
1671                         }
1672                     }
1673                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1674                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1675                             externalBgpVpnList, oldVpnListCopy, futures);
1676
1677                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1678                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1679                     //Identify the router VPN Instance and pass that value as oldVpnList
1680                     List<String> routerVpnList = new ArrayList<>();
1681                     for (String newVpnName : newVpnListCopy) {
1682                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1683                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1684                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1685                                 .BgpvpnType.VPN) {
1686                             routerVpnList.add(newVpnName);
1687                             break;
1688                         }
1689                     }
1690                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1691                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1692                             newVpnList, oldVpnListCopy, futures);
1693
1694                 } else {
1695                     //Handle remaining use cases.
1696                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1697                             oldVpnListCopy, futures);
1698                 }
1699             } else {
1700                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1701                         oldVpnListCopy, futures);
1702             }
1703         }
1704         return isVpnInstanceUpdate;
1705     }
1706
1707     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1708                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1709                                          List<String> newVpnList, List<String> oldVpnListCopy,
1710                                          List<ListenableFuture<Void>> futures) {
1711         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1712         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1713                 ? origAdjs.getAdjacency() : new ArrayList<>();
1714         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1715         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1716                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1717
1718         boolean isOldVpnRemoveCallExecuted = false;
1719         for (String oldVpnName : oldVpnList) {
1720             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1721                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1722             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1723             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1724                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1725                     newVpnList);
1726             isOldVpnRemoveCallExecuted = true;
1727         }
1728         //Wait for previous interface bindings to be removed
1729         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1730             try {
1731                 Thread.sleep(2000);
1732             } catch (InterruptedException e) {
1733                 //Ignore
1734             }
1735         }
1736         for (String newVpnName : newVpnList) {
1737             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1738             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1739                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1740                         + "onto vpnName {} ", interfaceName, newVpnName);
1741                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1742                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1743                                 + "from oldVpn(s) {} to newVpn {} ",
1744                         interfaceName, oldVpnListCopy, newVpnName);
1745                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1746                  * Use Case:
1747                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1748                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1749                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1750                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1751                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1752                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1753                  *     router VPN as well as Internet BGP-VPN.
1754                  *
1755                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1756                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1757                  *                                       update in existing router VPN instance.
1758                  */
1759                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1760                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1761                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1762                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1763                 }
1764             }
1765         }
1766     }
1767
1768     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1769                                                                     String vpnInterfaceName,
1770                                                                     List<ListenableFuture<Void>> futures) {
1771         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1772         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1773                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1774         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1775         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1776                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1777
1778         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1779         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1780             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1781             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1782             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1783             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1784             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1785                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1786                 //set of prefix used as entry in prefix-to-interface datastore
1787                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1788                 // in bucket actions on bgp-vpn delete
1789                 Set<String> prefixListForRefreshFib = new HashSet<>();
1790                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1791                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1792                         operTx -> {
1793                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1794                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1795                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1796                                     update.getName(), newVpnName);
1797                             //handle both addition and removal of adjacencies
1798                             // currently, new adjacency may be an extra route
1799                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1800                             if (!oldAdjs.equals(newAdjs)) {
1801                                 for (Adjacency adj : copyNewAdjs) {
1802                                     if (copyOldAdjs.contains(adj)) {
1803                                         copyOldAdjs.remove(adj);
1804                                     } else {
1805                                         // add new adjacency
1806                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1807                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1808                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1809                                         }
1810                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1811                                             + " added to vpn interface {} on vpn {} dpnId {}",
1812                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1813                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1814                                     }
1815                                 }
1816                                 for (Adjacency adj : copyOldAdjs) {
1817                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1818                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1819                                             && !adj.isPhysNetworkFunc()) {
1820                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1821                                                 confTx);
1822                                             //remove FIB entry
1823                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1824                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1825                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1826                                             //remove BGP entry
1827                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1828                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1829                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1830                                             }
1831                                         } else {
1832                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1833                                                 operTx, confTx);
1834                                         }
1835                                     }
1836                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1837                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1838                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1839                                 }
1840                             }
1841                         })));
1842                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1843                     MoreExecutors.directExecutor());
1844                 futures.add(configTxFuture);
1845                 for (ListenableFuture<Void> future : futures) {
1846                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1847                             update.getName(), update.getVpnInstanceNames());
1848                 }
1849             } else {
1850                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1851                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1852             }
1853         }
1854         return futures;
1855     }
1856
1857     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1858         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1859                 .toString();
1860         // FIXME: separate this out somehow?
1861         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1862         lock.lock();
1863         try {
1864             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1865                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1866             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1867                 LogicalDatastoreType.OPERATIONAL, lriIid);
1868             if (opResult.isPresent()) {
1869                 LabelRouteInfo labelRouteInfo =
1870                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1871                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1872                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1873             }
1874             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1875                     nextHopIpList);
1876         } catch (ReadFailedException e) {
1877             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1878                     nextHopIpList);
1879         } catch (TransactionCommitFailedException e) {
1880             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1881                     nextHopIpList);
1882         } finally {
1883             lock.unlock();
1884         }
1885     }
1886
1887     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1888         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1889
1890         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1891         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1892                 .addAugmentation(SubnetRoute.class, route).build();
1893         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1894         InstanceIdentifierBuilder<VrfTables> idBuilder =
1895             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1896         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1897         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1898         if (writeConfigTxn != null) {
1899             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1900         } else {
1901             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1902         }
1903         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1904                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1905     }
1906
1907     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1908                                            Adjacency adj, BigInteger dpnId,
1909                                            TypedWriteTransaction<Operational> writeOperTxn,
1910                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1911                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1912                                            Set<String> prefixListForRefreshFib)
1913             throws ExecutionException, InterruptedException {
1914         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1915         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1916         try {
1917             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1918                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1919             if (optVpnInterface.isPresent()) {
1920                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1921                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1922                 String vpnName = currVpnIntf.getVpnInstanceName();
1923                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1924                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1925                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1926                         LogicalDatastoreType.OPERATIONAL, adjPath);
1927                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1928                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1929                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1930                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1931                 List<Adjacency> adjacencies = new ArrayList<>();
1932                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1933                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1934                 }
1935                 long vpnId = vpnUtil.getVpnId(vpnName);
1936                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1937                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1938                 Adjacency operationalAdjacency = null;
1939                 //Handling dual stack neutron port primary adjacency
1940                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1941                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1942                             currVpnIntf.getName(), vpnName);
1943                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1944                             currVpnIntf.getName());
1945                     if (interfaceState != null) {
1946                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1947                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1948                             prefixListForRefreshFib);
1949                     }
1950                 }
1951                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1952                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1953                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1954                             : RouteOrigin.STATIC;
1955                     String nh = adj.getNextHopIpList().get(0);
1956                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1957                     // FIXME: separate out to somehow?
1958                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1959                     lock.lock();
1960                     try {
1961                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1962                                 vpnId, null, prefix, vpnName, nh, dpnId);
1963                         if (rdToAllocate.isPresent()) {
1964                             input.setRd(rdToAllocate.get());
1965                             operationalAdjacency = populator.createOperationalAdjacency(input);
1966                             int label = operationalAdjacency.getLabel().intValue();
1967                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1968                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1969                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1970                                     prefixListForRefreshFib, writeConfigTxn);
1971                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1972                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1973                                     vpnName, label, currVpnIntf.getName(), dpnId);
1974                         } else {
1975                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1976                                     vpnName, prefix);
1977                             return;
1978                         }
1979                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1980                         // Keeping the MPLS check for now.
1981                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1982                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1983                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1984                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1985                             vpnsToImportRoute.forEach(vpn -> {
1986                                 if (vpn.getVrfId() != null) {
1987                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1988                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1989                                             .ifPresent(
1990                                                 rds -> vpnManager.addExtraRoute(
1991                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1992                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1993                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1994                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1995                                 }
1996                             });
1997                         }
1998                     } finally {
1999                         lock.unlock();
2000                     }
2001                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2002                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2003                             currVpnIntf.getName(), vpnName);
2004
2005                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2006                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2007                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2008                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2009                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
2010                             Prefixes.PrefixCue.PhysNetFunc);
2011                     if (vpnIntefaceConfig.isPresent()) {
2012                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
2013                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2014                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
2015                     }
2016
2017                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2018
2019                     writeOperTxn.merge(
2020                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2021                                     prefix), pnfPrefix, true);
2022
2023                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2024                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
2025                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
2026
2027                     input.setRd(adj.getVrfId());
2028                 }
2029                 if (operationalAdjacency == null) {
2030                     operationalAdjacency = populator.createOperationalAdjacency(input);
2031                 }
2032                 adjacencies.add(operationalAdjacency);
2033                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2034                 VpnInterfaceOpDataEntry newVpnIntf =
2035                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2036                                 aug, dpnId, currVpnIntf.getLportTag(),
2037                                 currVpnIntf.getGatewayMacAddress());
2038
2039                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2040             }
2041         } catch (ReadFailedException e) {
2042             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2043                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2044         }
2045     }
2046
2047     @Nullable
2048     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2049         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2050         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2051     }
2052
2053     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2054                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2055                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2056         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2057         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2058         try {
2059             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2060                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2061             if (optVpnInterface.isPresent()) {
2062                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2063                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2064                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2065                         LogicalDatastoreType.OPERATIONAL, path);
2066                 if (optAdjacencies.isPresent()) {
2067                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2068
2069                     if (adjacencies != null && !adjacencies.isEmpty()) {
2070                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2071                         for (Adjacency adjacency : adjacencies) {
2072                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2073                                 String rd = adjacency.getVrfId();
2074                                 if (adj.getNextHopIpList() != null) {
2075                                     for (String nh : adj.getNextHopIpList()) {
2076                                         deleteExtraRouteFromCurrentAndImportingVpns(
2077                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2078                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2079                                     }
2080                                 } else if (adj.isPhysNetworkFunc()) {
2081                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2082                                             adj.getIpAddress(), adj.getSubnetId());
2083                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2084                                             writeConfigTxn);
2085                                 }
2086                                 break;
2087                             }
2088
2089                         }
2090                     }
2091                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2092                             dpnId, adj.getVrfId());
2093                 } else {
2094                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2095                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2096                 }
2097             }
2098         } catch (ReadFailedException e) {
2099             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2100                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2101         }
2102     }
2103
2104     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2105                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2106                                     TypedWriteTransaction<Operational> writeOperTx) {
2107         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2108         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2109         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2110             String vpnRd = vpn.getVrfId();
2111             if (vpnRd != null) {
2112                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2113                         writeOperTx);
2114             }
2115         }
2116     }
2117
2118     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2119         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2120             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2121             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2122     }
2123
2124     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2125         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2126             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2127     }
2128
2129     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2130                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2131         if (vpnInterface == null) {
2132             return;
2133         }
2134         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2135         if (adjs == null) {
2136             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2137                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2138             return;
2139         }
2140         for (Adjacency adj : adjs) {
2141             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2142                 String primaryInterfaceIp = adj.getIpAddress();
2143                 String macAddress = adj.getMacAddress();
2144                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2145
2146                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2147                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2148
2149                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2150                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2151                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2152                         routerInt, label, writeConfigTxn);
2153                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2154                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2155                         macAddress);
2156             } else {
2157                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2158                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2159                         interfaceName, primaryRd, vpnName);
2160             }
2161         }
2162     }
2163
2164     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2165             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2166         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2167         String rd = vpnUtil.getVpnRd(vpnName);
2168         if (adjs != null) {
2169             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2170             for (Adjacency adj : adjsList) {
2171                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2172                     String primaryInterfaceIp = adj.getIpAddress();
2173                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2174                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2175                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2176                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2177                 }
2178             }
2179         } else {
2180             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2181                     vpnInterface.getName(), rd);
2182         }
2183     }
2184
2185     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2186         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2187         final String interfaceName = key.getName();
2188         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2189                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2190         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2191                   intefaceData.identifier, vpnName);
2192     }
2193
2194     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2195                                               VpnInterface vpnInterface, String vpnName) {
2196         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2197                .get(vpnName);
2198         if (vpnInterfaces == null) {
2199             vpnInterfaces = new ConcurrentLinkedQueue<>();
2200         }
2201         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2202         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2203         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2204                 vpnInterface.getName(), vpnName);
2205     }
2206
2207     public boolean isVpnInstanceReady(String vpnInstanceName) {
2208         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2209         if (vpnRd == null) {
2210             return false;
2211         }
2212         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2213
2214         return vpnInstanceOpDataEntry != null;
2215     }
2216
2217     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2218         // FIXME: separate out to somehow?
2219         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2220         lock.lock();
2221         try {
2222             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2223                     unprocessedVpnInterfaces.get(vpnInstanceName);
2224             if (vpnInterfaces != null) {
2225                 while (!vpnInterfaces.isEmpty()) {
2226                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2227                     if (hasVpnInstanceCreatedSuccessfully) {
2228                         processSavedInterface(savedInterface, vpnInstanceName);
2229                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2230                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2231                     } else {
2232                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2233                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2234                     }
2235                 }
2236             } else {
2237                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2238             }
2239         } finally {
2240             lock.unlock();
2241         }
2242     }
2243
2244     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2245             VpnInterface vpnInterface) {
2246         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2247         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2248         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2249         lock.lock();
2250         try {
2251             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2252                     unprocessedVpnInterfaces.get(firstVpnName);
2253             if (vpnInterfaces != null) {
2254                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2255                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2256                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2257                 }
2258             } else {
2259                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2260             }
2261         } finally {
2262             lock.unlock();
2263         }
2264     }
2265
2266     public void vpnInstanceIsReady(String vpnInstanceName) {
2267         processSavedInterfaces(vpnInstanceName, true);
2268     }
2269
2270     public void vpnInstanceFailed(String vpnInstanceName) {
2271         processSavedInterfaces(vpnInstanceName, false);
2272     }
2273
2274     private static class UnprocessedVpnInterfaceData {
2275         InstanceIdentifier<VpnInterface> identifier;
2276         VpnInterface vpnInterface;
2277
2278         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2279             this.identifier = identifier;
2280             this.vpnInterface = vpnInterface;
2281         }
2282
2283         @Override
2284         public int hashCode() {
2285             final int prime = 31;
2286             int result = 1;
2287             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2288             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2289             return result;
2290         }
2291
2292         @Override
2293         public boolean equals(Object obj) {
2294             if (this == obj) {
2295                 return true;
2296             }
2297             if (obj == null) {
2298                 return false;
2299             }
2300             if (getClass() != obj.getClass()) {
2301                 return false;
2302             }
2303             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2304             if (identifier == null) {
2305                 if (other.identifier != null) {
2306                     return false;
2307                 }
2308             } else if (!identifier.equals(other.identifier)) {
2309                 return false;
2310             }
2311             if (vpnInterface == null) {
2312                 if (other.vpnInterface != null) {
2313                     return false;
2314                 }
2315             } else if (!vpnInterface.equals(other.vpnInterface)) {
2316                 return false;
2317             }
2318             return true;
2319         }
2320     }
2321
2322     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2323         String primaryRd = vpnUtil.getVpnRd(vpnName);
2324         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2325         if (vpnInstanceOpData == null) {
2326             return;
2327         }
2328         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2329         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2330             return;
2331         }
2332         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2333         vpnToDpnLists.forEach(vpnToDpnList -> {
2334             if (vpnToDpnList.getVpnInterfaces() == null) {
2335                 return;
2336             }
2337             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2338                 try {
2339                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2340                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2341                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2342                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2343                     if (!vpnInterfaceOptional.isPresent()) {
2344                         return;
2345                     }
2346                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2347                             vpnInterface.getInterfaceName());
2348                     if (configVpnAdjacencies == null) {
2349                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2350                         return;
2351                     }
2352                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2353                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2354                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2355                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2356                     // in operational DS. These unprocessed adjacencies will be handled below.
2357                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2358                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2359                     configVpnAdjacencies.stream()
2360                         .filter(adjacency -> operationVpnAdjacencies.stream()
2361                                 .noneMatch(operationalAdjacency ->
2362                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2363                         .forEach(adjacency -> {
2364                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2365                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2366                                 () -> {
2367                                     // TODO Deal with sequencing — the config tx must only submitted
2368                                     // if the oper tx goes in
2369                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2370                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2371                                         futures.add(
2372                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2373                                                 //set of prefix used, as entry in prefix-to-interface datastore
2374                                                 // is prerequisite for refresh Fib to avoid race condition leading
2375                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2376                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2377                                                 ListenableFuture<Void> configTxFuture =
2378                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2379                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2380                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2381                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2382                                                 Futures.addCallback(configTxFuture,
2383                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2384                                                     MoreExecutors.directExecutor());
2385                                                 futures.add(configTxFuture);
2386                                             }));
2387                                         return futures;
2388                                     } else {
2389                                         return emptyList();
2390                                     }
2391                                 });
2392                         });
2393                 } catch (ReadFailedException e) {
2394                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2395                             vpnName, primaryRd);
2396                 }
2397             });
2398         });
2399     }
2400
2401     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2402         private final String interfaceName;
2403         private final boolean add;
2404         private final String txnDestination;
2405
2406         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2407             this.interfaceName = interfaceName;
2408             this.add = add;
2409             this.txnDestination = transactionDest;
2410         }
2411
2412         @Override
2413         public void onSuccess(Void voidObj) {
2414             if (add) {
2415                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2416                         interfaceName, txnDestination);
2417             } else {
2418                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2419             }
2420         }
2421
2422         @Override
2423         public void onFailure(Throwable throwable) {
2424             if (add) {
2425                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2426                         interfaceName, txnDestination, throwable);
2427             } else {
2428                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2429                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2430             }
2431         }
2432     }
2433
2434     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2435         private final String primaryRd;
2436         private final Set<String> prefixListForRefreshFib;
2437
2438         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2439             this.primaryRd = primaryRd;
2440             this.prefixListForRefreshFib = prefixListForRefreshFib;
2441         }
2442
2443         @Override
2444         public void onSuccess(Void voidObj) {
2445             prefixListForRefreshFib.forEach(prefix -> {
2446                 fibManager.refreshVrfEntry(primaryRd, prefix);
2447             });
2448         }
2449
2450         @Override
2451         public void onFailure(Throwable throwable) {
2452             LOG.debug("write Tx config operation failed", throwable);
2453         }
2454     }
2455 }