vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.utils.JvmGlobalLocks;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 129
 130     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 131     private static final short DJC_MAX_RETRIES = 3;
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final IBgpManager bgpManager;
 136     private final IFibManager fibManager;
 137     private final IMdsalApiManager mdsalManager;
 138     private final IdManagerService idManager;
 139     private final OdlInterfaceRpcService ifaceMgrRpcService;
 140     private final VpnFootprintService vpnFootprintService;
 141     private final IInterfaceManager interfaceManager;
 142     private final IVpnManager vpnManager;
 143     private final ArpResponderHandler arpResponderHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final VpnUtil vpnUtil;
 146
 147     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 148
 149     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 150             new ConcurrentHashMap<>();
 151
 152     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 153
 154     @Inject
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final VpnFootprintService vpnFootprintService,
 162                                final IInterfaceManager interfaceManager,
 163                                final IVpnManager vpnManager,
 164                                final ArpResponderHandler arpResponderHandler,
 165                                final JobCoordinator jobCoordinator,
 166                                final CacheProvider cacheProvider,
 167                                final VpnUtil vpnUtil) {
 168         super(VpnInterface.class, VpnInterfaceManager.class);
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187     }
 188
 189     public Runnable isNotifyTaskQueued(String intfName) {
 190         return vpnIntfMap.remove(intfName);
 191     }
 192
 193     @PostConstruct
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204     }
 205
 206     @Override
 207     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 208         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 209     }
 210
 211     @Override
 212     protected VpnInterfaceManager getDataTreeChangeListener() {
 213         return VpnInterfaceManager.this;
 214     }
 215
 216     @Override
 217     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 218         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 219         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 220                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 221         addVpnInterface(identifier, vpnInterface, null, null);
 222     }
 223
 224     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 225                           final VpnInterface vpnInterface, String vpnName) {
 226         // FIXME: separate this out somehow?
 227         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 228         lock.lock();
 229         try {
 230             if (isVpnInstanceReady(vpnName)) {
 231                 return true;
 232             }
 233             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 234             return false;
 235         } finally {
 236             lock.unlock();
 237         }
 238     }
 239
 240     // TODO Clean up the exception handling
 241     @SuppressWarnings("checkstyle:IllegalCatch")
 242     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 244         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 245             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 246             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 247         }
 248     }
 249
 250     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 251                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 252         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 253         final String interfaceName = key.getName();
 254
 255         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 256             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 257                   vpnName, vpnInterface.getName());
 258             return;
 259         }
 260         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 261              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 262         List<Adjacency> copyOldAdjs = null;
 263         if (oldAdjs != null) {
 264             copyOldAdjs = new ArrayList<>();
 265             copyOldAdjs.addAll(oldAdjs);
 266         }
 267         List<Adjacency> copyNewAdjs = null;
 268         if (newAdjs != null) {
 269             copyNewAdjs = new ArrayList<>();
 270             copyNewAdjs.addAll(newAdjs);
 271         }
 272         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 273     }
 274
 275     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 276                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 277                     final @Nullable List<Adjacency> newAdjs,
 278                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 279         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 280         final String interfaceName = key.getName();
 281         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 282         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 283             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 284             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 285             if (interfaceState != null) {
 286                 try {
 287                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 288                     final int ifIndex = interfaceState.getIfIndex();
 289                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 290                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 291                         // (the inventory tx goes in last)
 292                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 293                         //set of prefix used, as entry in prefix-to-interface datastore
 294                         // is prerequisite for refresh Fib to avoid race condition leading to
 295                         // missing remote next hop in bucket actions on bgp-vpn delete
 296                         Set<String> prefixListForRefreshFib = new HashSet<>();
 297                         ListenableFuture<Void> confFuture =
 298                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 299                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 300                                     operTx -> futures.add(
 301                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 302                                             LOG.info(
 303                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 304                                                     + " on dpn {}",
 305                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 306                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 307                                                 confTx, operTx, invTx, interfaceState, vpnName,
 308                                                 prefixListForRefreshFib);
 309                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 310                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 311                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 312                                                 if (newAdjs != null) {
 313                                                     for (Adjacency adj : newAdjs) {
 314                                                         if (oldAdjs.contains(adj)) {
 315                                                             oldAdjs.remove(adj);
 316                                                         } else {
 317                                                             if (!isBgpVpnInternetVpn
 318                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 320                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 321                                                                     prefixListForRefreshFib);
 322                                                             }
 323                                                         }
 324                                                     }
 325                                                 }
 326                                                 for (Adjacency adj : oldAdjs) {
 327                                                     if (!isBgpVpnInternetVpn
 328                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 329                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 330                                                             operTx, confTx);
 331                                                     }
 332                                                 }
 333                                             }
 334                                         })))));
 335                         Futures.addCallback(confFuture,
 336                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 337                             MoreExecutors.directExecutor());
 338                         futures.add(confFuture);
 339                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 340                             MoreExecutors.directExecutor());
 341                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 342                             + " processed successfully", interfaceName, vpnName, dpnId);
 343                         return futures;
 344                     });
 345                 } catch (NumberFormatException | IllegalStateException e) {
 346                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 347                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 348                         vpnName, e);
 349                     return;
 350                 }
 351             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 352                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 353                     () -> {
 354                         ListenableFuture<Void> future =
 355                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 356                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 357                                     confTx, vpnName);
 358                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 359                                     vpnName, vpnInterface.getDpnId());
 360                             });
 361                         ListenableFutures.addErrorLogging(future, LOG,
 362                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 363                         return Collections.singletonList(future);
 364                     });
 365             } else {
 366                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 367                     + " is not available", interfaceName, vpnName);
 368             }
 369         } else {
 370             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 371                     + " as vpn is pending delete", interfaceName, vpnName,
 372                 vpnInterface.getDpnId());
 373         }
 374     }
 375
 376     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 377     // see comments below.
 378     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 379     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 380             final int lportTag, boolean isInterfaceUp,
 381             TypedWriteTransaction<Configuration> writeConfigTxn,
 382             TypedWriteTransaction<Operational> writeOperTxn,
 383             TypedReadWriteTransaction<Configuration> writeInvTxn,
 384             Interface interfaceState, final String vpnName,
 385             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 386         final String interfaceName = vpnInterface.getName();
 387         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 388                 vpnName);
 389         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 390         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 391         if (!isInterfaceUp) {
 392             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 393                      interfaceName, dpId, vpnName);
 394             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 395             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 396                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 397                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 398                         dpId);
 399                 return;
 400             }
 401
 402             boolean waitForVpnInterfaceOpRemoval = false;
 403             if (opVpnInterface != null) {
 404                 String opVpnName = opVpnInterface.getVpnInstanceName();
 405                 String primaryInterfaceIp = null;
 406                 if (Objects.equals(opVpnName, vpnName)) {
 407                     // Please check if the primary VRF Entry does not exist for VPNInterface
 408                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 409                     // back to back
 410                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 411                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 412                     if (adjs == null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 414                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 415                                 vpnName);
 416                         return;
 417                     }
 418                     for (Adjacency adj : adjs) {
 419                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 420                             primaryInterfaceIp = adj.getIpAddress();
 421                             break;
 422                         }
 423                     }
 424                     if (primaryInterfaceIp == null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 426                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 427                                 dpId, vpnName);
 428                         return;
 429                     }
 430                     // Get the rd of the vpn instance
 431                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 432                     if (vrf != null) {
 433                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 434                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 435                         return;
 436                     }
 437                     waitForVpnInterfaceOpRemoval = true;
 438                 } else {
 439                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 440                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 441                 }
 442             }
 443             if (!waitForVpnInterfaceOpRemoval) {
 444                 // Add the VPNInterface and quit
 445                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 446                         null/*ipAddressSourceValuePair*/,
 447                         true /* add */);
 448                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 449                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 450                 if (!isBgpVpnInternetVpn) {
 451                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 452                 }
 453                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 454                         dpId, vpnName);
 455                 if (interfaceManager.isExternalInterface(interfaceName)) {
 456                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 457                         NwConstants.ADD_FLOW);
 458                 }
 459                 return;
 460             }
 461
 462             // FIB didn't get a chance yet to clean up this VPNInterface
 463             // Let us give it a chance here !
 464             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 465                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 466             try {
 467                 Runnable notifyTask = new VpnNotifyTask();
 468                 synchronized (notifyTask) {
 469                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 470                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 471                     // published here while holding the lock on it so this path will hit the wait before notify can be
 472                     // invoked.
 473                     vpnIntfMap.put(interfaceName, notifyTask);
 474                     try {
 475                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 476                     } catch (InterruptedException e) {
 477                         // Ignored
 478                     }
 479                 }
 480             } finally {
 481                 vpnIntfMap.remove(interfaceName);
 482             }
 483
 484             if (opVpnInterface != null) {
 485                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 486                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 487                         dpId, vpnName);
 488                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 489                 return;
 490             }
 491             // VPNInterface got removed, proceed with Add
 492             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 493                     interfaceName, dpId, vpnName);
 494             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 495                     null/*ipAddressSourceValuePair*/,
 496                     true /* add */);
 497             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 498                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 499             if (!isBgpVpnInternetVpn) {
 500                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 501             }
 502             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 503                     + " FIB to clean up", interfaceName, dpId, vpnName);
 504             if (interfaceManager.isExternalInterface(interfaceName)) {
 505                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 506                                lportTag, NwConstants.ADD_FLOW);
 507             }
 508         } else {
 509             try {
 510                 // Interface is retained in the DPN, but its Link Up.
 511                 // Advertise prefixes again for this interface to BGP
 512                 InstanceIdentifier<VpnInterface> identifier =
 513                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 514                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 515                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 516                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 517                 // Perform similar operation as interface add event for extraroutes.
 518                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 519                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 520                         LogicalDatastoreType.CONFIGURATION, path);
 521                 if (!optAdjacencies.isPresent()) {
 522                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 523                     return;
 524                 }
 525                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 526                 for (Adjacency adjacency : adjacencies) {
 527                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 528                         continue;
 529                     }
 530                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 531                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 532                         continue;
 533                     }
 534                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 535                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 536                 }
 537             } catch (ReadFailedException e) {
 538                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 539                         interfaceName, vpnName, primaryRd, dpId);
 540             }
 541         }
 542     }
 543
 544     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 545         int lportTag, int addOrRemove) {
 546         Uuid extNetworkId;
 547         try {
 548             // vpn instance of ext-net interface is the network-id
 549             extNetworkId = new Uuid(vpnName);
 550         } catch (IllegalArgumentException e) {
 551             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 552                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 553             return;
 554         }
 555
 556         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 557         if (routerIds == null || routerIds.isEmpty()) {
 558             LOG.info("processExternalVpnInterface: No router is associated with {}."
 559                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 560                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 561             return;
 562         }
 563
 564         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 565                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 566         for (Uuid routerId : routerIds) {
 567             String routerName = routerId.getValue();
 568             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 569             if (Objects.equals(primarySwitch, dpId)) {
 570                 Routers router = vpnUtil.getExternalRouter(routerName);
 571                 if (router != null) {
 572                     if (addOrRemove == NwConstants.ADD_FLOW) {
 573                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 574                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 575                                 dpId, interfaceName, lportTag);
 576                     } else {
 577                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 578                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 579                                 dpId, interfaceName, lportTag);
 580                     }
 581                 } else {
 582                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 583                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 584                             interfaceName, dpId, vpnName);
 585                 }
 586             }
 587         }
 588     }
 589
 590     // TODO Clean up the exception handling
 591     @SuppressWarnings("checkstyle:IllegalCatch")
 592     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 593                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 594                                                   String vpnName, String interfaceName) {
 595         if (rd == null) {
 596             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 597                   interfaceName, dpnId, vpnName);
 598             return;
 599         } else {
 600             if (rd.equals(vpnName)) {
 601                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 602                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 603                 return;
 604             }
 605         }
 606         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 607                 interfaceName, dpnId, vpnName, rd);
 608
 609         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 610         if (nextHopIp == null) {
 611             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 612                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 613                     rd, vpnName);
 614             return;
 615         }
 616
 617         try {
 618             //Read NextHops
 619             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 620             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 621                     LogicalDatastoreType.OPERATIONAL, path);
 622             if (adjacencies.isPresent()) {
 623                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 624                 if (nextHops != null && !nextHops.isEmpty()) {
 625                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 626                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 627                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 628                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 629                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 630                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 631                     for (Adjacency nextHop : nextHops) {
 632                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 633                             continue;
 634                         }
 635                         String gatewayMac = null;
 636                         Uint32 label = Uint32.ZERO;
 637                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 638                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 639                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 640                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 641                                     .get();
 642                         } else {
 643                             label = nextHop.getLabel();
 644                         }
 645                         try {
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 647                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 648                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 649                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 650                                     gatewayMac);
 651                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 652                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 653                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 654                         } catch (Exception e) {
 655                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 656                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 657                                     interfaceName, dpnId, e);
 658                         }
 659                     }
 660                 }
 661             }
 662         } catch (ReadFailedException e) {
 663             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 664                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 665         }
 666     }
 667
 668     // TODO Clean up the exception handling
 669     @SuppressWarnings("checkstyle:IllegalCatch")
 670     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 671                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 672                         TypedWriteTransaction<Operational> writeOperTx) {
 673         //Read NextHops
 674         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 675         String rd = vpnUtil.getVpnRd(interfaceName);
 676         if (rd == null) {
 677             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 678                 interfaceName, vpnName);
 679             return;
 680         } else {
 681             if (rd.equals(vpnName)) {
 682                 LOG.info(
 683                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 684                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 685                 return;
 686             }
 687         }
 688         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 689                vpnName, rd);
 690         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 691         try {
 692             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 693                     path);
 694         } catch (ReadFailedException e) {
 695             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 696                     interfaceName, vpnName);
 697         }
 698         if (adjacencies.isPresent()) {
 699             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 700
 701             if (nextHops != null && !nextHops.isEmpty()) {
 702                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 703                         nextHops, interfaceName, vpnName, rd);
 704                 for (Adjacency nextHop : nextHops) {
 705                     try {
 706                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 707                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 708                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 709                                     interfaceName, vpnName);
 710                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 711                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 712                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 713                                     interfaceName, vpnName);
 714                         } else if (nextHop.getNextHopIpList() != null) {
 715                             // Perform similar operation as interface delete event for extraroutes.
 716                             String allocatedRd = nextHop.getVrfId();
 717                             for (String nh : nextHop.getNextHopIpList()) {
 718                                 deleteExtraRouteFromCurrentAndImportingVpns(
 719                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 720                                     writeOperTx);
 721                             }
 722                         }
 723                     } catch (Exception e) {
 724                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 725                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 726                     }
 727                 }
 728             }
 729         }
 730     }
 731
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 734                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 735                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 736                                                   TypedWriteTransaction<Operational> writeOperTxn,
 737                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 738                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 739             throws ExecutionException, InterruptedException {
 740         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 741         // Read NextHops
 742         Optional<VpnInterface> vpnInteface = Optional.absent();
 743         try {
 744             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 745             LogicalDatastoreType.CONFIGURATION, identifier);
 746         } catch (ReadFailedException e) {
 747             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 748                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 749         }
 750         Uuid intfnetworkUuid = null;
 751         NetworkType networkType = null;
 752         Long segmentationId = Long.valueOf(-1);
 753         Adjacencies adjacencies = null;
 754         if (vpnInteface.isPresent()) {
 755             intfnetworkUuid = vpnInteface.get().getNetworkId();
 756             networkType = vpnInteface.get().getNetworkType();
 757             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 758             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 759             if (adjacencies == null) {
 760                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 761                         null/*gwMac*/,  null/*gatewayIp*/, writeOperTxn);
 762                 return;
 763             }
 764         }
 765         // Get the rd of the vpn instance
 766         String nextHopIp = null;
 767         String gatewayIp = null;
 768         try {
 769             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 770         } catch (Exception e) {
 771             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 772                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 773         }
 774         List<String> nhList = new ArrayList<>();
 775         if (nextHopIp != null) {
 776             nhList.add(nextHopIp);
 777             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 778                     interfaceName, dpnId, vpnName, nhList);
 779         }
 780         Optional<String> gwMac = Optional.absent();
 781         String vpnInterfaceSubnetGwMacAddress = null;
 782         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 783         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 784         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 785         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 786         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 787         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 788         List<Adjacency> value = new ArrayList<>();
 789         for (Adjacency nextHop : nextHops) {
 790             String rd = primaryRd;
 791             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 792             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 793                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 794                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 795                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 796                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 797                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 798                 continue;
 799             }
 800             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 801                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 802                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 803                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 804                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 805                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 806
 807                 Prefixes prefixes = intfnetworkUuid != null
 808                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 809                             segmentationId, prefixCue) :
 810                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 811                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 812                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 813                 final Uuid subnetId = nextHop.getSubnetId();
 814
 815                 gatewayIp = nextHop.getSubnetGatewayIp();
 816                 if (gatewayIp == null) {
 817                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 818                     if (gatewayIpOptional.isPresent()) {
 819                         gatewayIp = gatewayIpOptional.get();
 820                     }
 821                 }
 822
 823                 if (gatewayIp != null) {
 824                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 825                     if (gwMac.isPresent()) {
 826                         // A valid mac-address is available for this subnet-gateway-ip
 827                         // Use this for programming ARP_RESPONDER table here.  And save this
 828                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 829                         // to populate L3_GW_MAC_TABLE there.
 830                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 831                                 gatewayIp, gwMac.get());
 832                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 833                     } else {
 834                         // A valid mac-address is not available for this subnet-gateway-ip
 835                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 836                         // Save this connected-mac-address as gateway-mac-address for the
 837                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 838                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 839                         if (gwMac.isPresent()) {
 840                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 841                                     NwConstants.ADD_FLOW, gwMac.get());
 842                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 843                                     gatewayIp, gwMac.get());
 844                         } else {
 845                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 846                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 847                                 + "gwIp {}",
 848                                 subnetId, interfaceName, vpnName, gatewayIp);
 849                         }
 850                     }
 851                 } else {
 852                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 853                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 854                         subnetId, interfaceName, vpnName);
 855                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 856                 }
 857                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 858                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 859             } else {
 860                 //Extra route adjacency
 861                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 862                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 863                 // FIXME: separate this out somehow?
 864                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 865                 lock.lock();
 866                 try {
 867                     java.util.Optional<String> rdToAllocate = vpnUtil
 868                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 869                                     nextHop.getNextHopIpList().get(0), dpnId);
 870                     if (rdToAllocate.isPresent()) {
 871                         rd = rdToAllocate.get();
 872                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 873                             rd, prefix);
 874                     } else {
 875                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 876                         continue;
 877                     }
 878                 } finally {
 879                     lock.unlock();
 880                 }
 881                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 882                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 883                         interfaceName, dpnId);
 884             }
 885             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 886             // can be different from the gateway-mac-address within the VRFEntry as the
 887             // gateway-mac-address is a superset.
 888             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 889             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 890                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 891                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 892             Adjacency operationalAdjacency = null;
 893             try {
 894                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 895             } catch (NullPointerException e) {
 896                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 897                     input, e.getMessage());
 898                 return;
 899             }
 900             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 901                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 902                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 903                     writeConfigTxn);
 904             }
 905             value.add(operationalAdjacency);
 906         }
 907
 908         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 909         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 910                 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
 911
 912         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 913                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 914                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 915
 916         for (Adjacency nextHop : aug.getAdjacency()) {
 917             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 918             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 919                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 920                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 921                 registeredPopulator.populateFib(input, writeConfigTxn);
 922             }
 923         }
 924     }
 925
 926     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 927                                               long lportTag, String gwMac, String gwIp,
 928                                               TypedWriteTransaction<Operational> writeOperTxn) {
 929         VpnInterfaceOpDataEntry opInterface =
 930               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
 931         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 932             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 933         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 934         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 935                 interfaceName, dpnId, vpnName);
 936     }
 937
 938     // TODO Clean up the exception handling
 939     @SuppressWarnings("checkstyle:IllegalCatch")
 940     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 941                                            StateTunnelList stateTunnelList,
 942                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 943                                            TypedWriteTransaction<Operational> writeOperTxn) {
 944
 945         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 946         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 947         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 948         List<Adjacency> adjList =
 949             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 950         if (adjList.isEmpty()) {
 951             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 952                     vpnInterface, srcDpnId);
 953             return;
 954         }
 955         String prefix = null;
 956         List<Adjacency> value = new ArrayList<>();
 957         boolean isNextHopAddReqd = false;
 958         String vpnName = vpnInterface.getVpnInstanceName();
 959         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 960         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 961         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 962                 vpnInterface.getName(), vpnInterface.getDpnId(),
 963                 vpnInterface.getVpnInstanceName(), adjList);
 964         for (Adjacency adj : adjList) {
 965             String rd = adj.getVrfId();
 966             rd = rd != null ? rd : vpnName;
 967             prefix = adj.getIpAddress();
 968             Uint32 label = adj.getLabel();
 969             List<String> nhList = Collections.singletonList(srcTepIp);
 970             List<String> nextHopList = adj.getNextHopIpList();
 971             // If TEP is added , update the nexthop of primary adjacency.
 972             // Secondary adj nexthop is already pointing to primary adj IP address.
 973             if (nextHopList == null || nextHopList.isEmpty()) {
 974                 isNextHopAddReqd = true;
 975             }
 976
 977             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 978                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 979             } else {
 980                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 981                 if (!vrfEntryOptional.isPresent()) {
 982                     continue;
 983                 }
 984                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 985                 if (!nhList.contains(srcTepIp)) {
 986                     nhList.add(srcTepIp);
 987                     isNextHopAddReqd = true;
 988                 }
 989                 value.add(adj);
 990             }
 991
 992             if (isNextHopAddReqd) {
 993                 updateLabelMapper(label, nhList);
 994                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 995                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 996                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 997                 // Update the VRF entry with nextHop
 998                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 999                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1000
1001                 //Get the list of VPN's importing this route(prefix) .
1002                 // Then update the VRF entry with nhList
1003                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1004                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1005                     String vpnRd = vpn.getVrfId();
1006                     if (vpnRd != null) {
1007                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1008                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1009                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1010                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1011                             vpnInterface.getName(), srcDpnId, vpnName,
1012                             vpn.getVpnInstanceName(), vpnRd);
1013                     }
1014                 }
1015                 // Advertise the prefix to BGP only for external vpn
1016                 // since there is a nexthop change.
1017                 try {
1018                     if (!rd.equalsIgnoreCase(vpnName)) {
1019                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1020                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1021                                 null /*gatewayMacAddress*/);
1022                     }
1023                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1024                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1025                             srcDpnId, vpnName);
1026                 } catch (Exception ex) {
1027                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1028                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1029                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1030                 }
1031             }
1032         }
1033         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1034         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1035                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1036                 .addAugmentation(AdjacenciesOp.class, aug).build();
1037         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1038                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1039         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1040         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1041                 vpnInterface.getName(), srcDpnId, vpnName);
1042
1043     }
1044
1045     // TODO Clean up the exception handling
1046     @SuppressWarnings("checkstyle:IllegalCatch")
1047     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1048                                               StateTunnelList stateTunnelList,
1049                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1050                                               TypedWriteTransaction<Operational> writeOperTxn) {
1051
1052         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1053         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1054         String prefix = null;
1055         boolean isNextHopRemoveReqd = false;
1056         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1057         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1058         String vpnName = vpnInterface.getVpnInstanceName();
1059         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1060         String primaryRd = vpnUtil.getVpnRd(vpnName);
1061         if (adjList != null) {
1062             List<Adjacency> value = new ArrayList<>();
1063             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1064                     vpnInterface.getName(), vpnInterface.getDpnId(),
1065                     vpnInterface.getVpnInstanceName(), adjList);
1066             for (Adjacency adj : adjList) {
1067                 List<String> nhList = new ArrayList<>();
1068                 String rd = adj.getVrfId();
1069                 rd = rd != null ? rd : vpnName;
1070                 prefix = adj.getIpAddress();
1071                 List<String> nextHopList = adj.getNextHopIpList();
1072                 Uint32 label = adj.getLabel();
1073                 if (nextHopList != null && !nextHopList.isEmpty()) {
1074                     isNextHopRemoveReqd = true;
1075                 }
1076                 // If TEP is deleted , remove the nexthop from primary adjacency.
1077                 // Secondary adj nexthop will continue to point to primary adj IP address.
1078                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1079                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1080                 } else {
1081                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1082                     if (!vrfEntryOptional.isPresent()) {
1083                         continue;
1084                     }
1085                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1086                     if (nhList.contains(srcTepIp)) {
1087                         nhList.remove(srcTepIp);
1088                         isNextHopRemoveReqd = true;
1089                     }
1090                     value.add(adj);
1091                 }
1092
1093                 if (isNextHopRemoveReqd) {
1094                     updateLabelMapper(label, nhList);
1095                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1096                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1097                             prefix, nhList, vpnName,
1098                             vpnId, rd, vpnInterface.getName());
1099                     // Update the VRF entry with removed nextHop
1100                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1101                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1102
1103                     //Get the list of VPN's importing this route(prefix) .
1104                     // Then update the VRF entry with nhList
1105                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1106                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1107                         String vpnRd = vpn.getVrfId();
1108                         if (vpnRd != null) {
1109                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1110                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1111                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1112                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1113                                     nhList, label, vpnInterface.getName(), srcDpnId,
1114                                     vpnName,
1115                                     vpn.getVpnInstanceName(), vpnRd);
1116                         }
1117                     }
1118
1119                     // Withdraw prefix from BGP only for external vpn.
1120                     try {
1121                         if (!rd.equalsIgnoreCase(vpnName)) {
1122                             bgpManager.withdrawPrefix(rd, prefix);
1123                         }
1124                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1125                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1126                                 vpnInterface.getName(), srcDpnId,
1127                                 vpnName);
1128                     } catch (Exception ex) {
1129                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1130                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1131                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1132                     }
1133                 }
1134             }
1135             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1136             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1137                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1138                     .addAugmentation(AdjacenciesOp.class, aug).build();
1139             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1140                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1141             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1142             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1143                          vpnInterface.getName(), srcDpnId, vpnName);
1144         }
1145     }
1146
1147     @SuppressWarnings("checkstyle:IllegalCatch")
1148     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1149         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1150         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry;
1151         String vpnRd = vpnUtil.getVpnRd(vpnName);
1152         try {
1153             VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1154             if (opDataEntry == null) {
1155                 LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}"
1156                         + " when check for vpns exporting the routes", vpnName, vpnRd);
1157                 return vpnsToExportRoute;
1158             }
1159             vpnInstanceOpDataEntry = opDataEntry;
1160         } catch (Exception re) {
1161             LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}"
1162                     + " to check for vpns exporting the routes", vpnName, vpnRd, re);
1163             return vpnsToExportRoute;
1164         }
1165         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1166             if (input.getVpnInstanceName() == null) {
1167                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1168                         input.getVrfId());
1169                 return false;
1170             }
1171             return !input.getVpnInstanceName().equals(vpnName);
1172         };
1173
1174         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1175             Iterable<String> commonRTs =
1176                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1177                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1178             return Iterators.size(commonRTs.iterator()) > 0;
1179         };
1180
1181         vpnsToExportRoute =
1182                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1183                         Collectors.toList());
1184         return vpnsToExportRoute;
1185     }
1186
1187     // TODO Clean up the exception handling
1188     @SuppressWarnings("checkstyle:IllegalCatch")
1189     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1190         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1191         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1192             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1193             if (vrfEntries != null) {
1194                 ListenableFutures.addErrorLogging(
1195                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1196                         for (VrfEntry vrfEntry : vrfEntries) {
1197                             try {
1198                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1199                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1200                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1201                                             + " is not a controller managed non intervpn link route. Ignoring.",
1202                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1203                                     continue;
1204                                 }
1205                                 String prefix = vrfEntry.getDestPrefix();
1206                                 String gwMac = vrfEntry.getGatewayMacAddress();
1207                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1208                                     String nh = routePath.getNexthopAddress();
1209                                     Uint32 label = routePath.getLabel();
1210                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1211                                         vrfEntry.getOrigin()))) {
1212                                         LOG.info(
1213                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1214                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1215                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1216                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1217                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1218                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1219                                             confTx);
1220                                     } else {
1221                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1222                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1223                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1224                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1225                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1226                                             confTx);
1227                                     }
1228                                 });
1229                             } catch (RuntimeException e) {
1230                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1231                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1232                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1233                             }
1234                         }
1235                     }), LOG, "Error handing VPN exporting routes");
1236             } else {
1237                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1238                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1239             }
1240         }
1241     }
1242
1243     @Override
1244     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1245         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1246         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1247         final String interfaceName = key.getName();
1248         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1249             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1250             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1251         }
1252     }
1253
1254     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1255                                 final VpnInterface vpnInterface, final String vpnName,
1256                                 final String interfaceName) {
1257         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1258             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1259                 ListenableFuture<Void> future =
1260                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1261                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1262                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1263                     });
1264                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1265                         vpnInterface.getName(), vpnName);
1266                 return Collections.singletonList(future);
1267             }, DJC_MAX_RETRIES);
1268         } else {
1269             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1270             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1271         }
1272     }
1273
1274     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1275     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1276                                            final VpnInterface vpnInterface, final String vpnName,
1277                                            final String interfaceName, final Interface interfaceState) {
1278         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1279                 vpnName, vpnInterface.getDpnId());
1280         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1281         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1282             () -> {
1283                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1284                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1285                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1286                         writeOperTxn -> futures.add(
1287                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1288                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1289                                         interfaceName, vpnName);
1290                                 Uint64 dpId;
1291                                 int ifIndex;
1292                                 String gwMacAddress;
1293                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1294                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1295                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1296                                 try {
1297                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1298                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1299                                 } catch (ReadFailedException e) {
1300                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1301                                             interfaceName, vpnName);
1302                                     return;
1303                                 }
1304                                 if (interfaceState != null) {
1305                                     try {
1306                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1307                                     } catch (NumberFormatException | IllegalStateException e) {
1308                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1309                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1310                                                         + " from vpn interface op data store. ", interfaceName,
1311                                                 vpnInterface.getDpnId(), vpnName, e);
1312                                         dpId = Uint64.ZERO;
1313                                     }
1314                                     ifIndex = interfaceState.getIfIndex();
1315                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1316                                 } else {
1317                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1318                                             + " from vpn interface op.", interfaceName);
1319                                     if (optVpnInterface.isPresent()) {
1320                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1321                                         dpId = vpnOpInterface.getDpnId();
1322                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1323                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1324                                     } else {
1325                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1326                                                 + " as interfaceState and vpn interface op is not"
1327                                                 + " available", interfaceName, vpnName);
1328                                         return;
1329                                     }
1330                                 }
1331                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1332                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1333                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1334                                 LOG.info(
1335                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1336                                                 + "successfully",
1337                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1338                             })))));
1339                 futures.add(configFuture);
1340                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1341                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1342                 return futures;
1343             }, DJC_MAX_RETRIES);
1344     }
1345
1346     protected void processVpnInterfaceDown(Uint64 dpId,
1347                                            String interfaceName,
1348                                            int lportTag,
1349                                            String gwMac,
1350                                            VpnInterfaceOpDataEntry vpnOpInterface,
1351                                            boolean isInterfaceStateDown,
1352                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1353                                            TypedWriteTransaction<Operational> writeOperTxn,
1354                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1355             throws ExecutionException, InterruptedException {
1356         if (vpnOpInterface == null) {
1357             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1358                     + " as it is not available in operational data store", interfaceName, dpId);
1359             return;
1360         }
1361         final String vpnName = vpnOpInterface.getVpnInstanceName();
1362         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1363                                                     interfaceName, vpnName);
1364         if (!isInterfaceStateDown) {
1365             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1366             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1367             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1368             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1369                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1370             if (interfaceManager.isExternalInterface(interfaceName)) {
1371                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1372                     NwConstants.DEL_FLOW);
1373             }
1374             if (!isBgpVpnInternetVpn) {
1375                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1376             }
1377             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1378                     + " successful", interfaceName, dpId, vpnName);
1379         } else {
1380             // Interface is retained in the DPN, but its Link Down.
1381             // Only withdraw the prefixes for this interface from BGP
1382             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1383         }
1384     }
1385
1386     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1387                                           final String vpnName, final Uint32 vpnId, String gwMac,
1388                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1389                                           TypedWriteTransaction<Operational> writeOperTxn,
1390                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1391             throws ExecutionException, InterruptedException {
1392         //Read NextHops
1393         try {
1394             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1395                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1396             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1397                     SingleTransactionDataBroker.syncReadOptional(dataBroker,
1398                             LogicalDatastoreType.OPERATIONAL, identifier);
1399             boolean isLearntIP = Boolean.FALSE;
1400             String primaryRd = vpnUtil.getVpnRd(vpnName);
1401             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1402                     interfaceName, dpnId, vpnName, primaryRd);
1403             if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1404                 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1405                         + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1406                 return;
1407             }
1408             AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1409
1410             if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) {
1411                 List<Adjacency> nextHops = adjacencies.getAdjacency();
1412                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1413                         interfaceName, dpnId, vpnName, nextHops);
1414                 for (Adjacency nextHop : nextHops) {
1415                     if (nextHop.isPhysNetworkFunc()) {
1416                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1417                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1418                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1419                                 null, null/*writeCfgTxn*/);
1420                     } else {
1421                         String rd = nextHop.getVrfId();
1422                         List<String> nhList;
1423                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1424                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1425                             isLearntIP = nextHop.getAdjacencyType() == AdjacencyType.LearntIp ? true : false;
1426                         } else {
1427                             // This is a primary adjacency
1428                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1429                                     : emptyList();
1430                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1431                                     vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1432                                     interfaceName, writeInvTxn);
1433                         }
1434                         if (!nhList.isEmpty()) {
1435                             if (Objects.equals(primaryRd, vpnName)) {
1436                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1437                                 //remove from FIB directly
1438                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1439                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1440                             } else {
1441                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1442                                         interfaceName, writeConfigTxn, writeOperTxn);
1443                             }
1444                         } else {
1445                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1446                                             + " interface {}", nextHop.getIpAddress(), rd,
1447                                     nextHop.getAdjacencyType().toString(), interfaceName);
1448                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1449                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1450                         }
1451                     }
1452                     String ip = nextHop.getIpAddress().split("/")[0];
1453                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1454                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1455                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1456                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1457                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1458                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1459                     }
1460                     // Remove the MIP-IP from VpnPortIpToPort.
1461                     if (isLearntIP) {
1462                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1463                         if (persistedIp != null && persistedIp.isLearntIp()
1464                                 && persistedIp.getPortName().equals(interfaceName)) {
1465                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1466                             LOG.info(
1467                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1468                                             + "from VpnPortipToPort",
1469                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1470                         }
1471                     }
1472                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1473                     if (vpnPortipToPort != null) {
1474                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1475                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1476                                  + "Interface {} ip {} on dpn {} for vpn {}",
1477                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1478                     }
1479                 }
1480             } else {
1481                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1482                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1483                         + " Removing it.", interfaceName, vpnName, dpnId);
1484                 writeOperTxn.delete(identifier);
1485             }
1486         } catch (ReadFailedException e) {
1487             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1488                     interfaceName, dpnId, vpnName);
1489         }
1490     }
1491
1492     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1493                                                             String interfaceName, Uint64 dpnId,
1494                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1495                                                             TypedWriteTransaction<Operational> writeOperTx) {
1496         return (nh) -> {
1497             String primaryRd = vpnUtil.getVpnRd(vpnName);
1498             String prefix = nextHop.getIpAddress();
1499             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1500             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1501                     nextHop, vpnName, interfaceName, dpnId);
1502             // FIXME: separate this out somehow?
1503             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1504             lock.lock();
1505             try {
1506                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1507                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1508                     //If extra-route is present behind at least one VM, then do not remove or update
1509                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1510                     //prefixtointerface DS
1511                     return;
1512                 }
1513                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1514                         writeConfigTxn);
1515             } finally {
1516                 lock.unlock();
1517             }
1518             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1519                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1520                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1521         };
1522     }
1523
1524     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1525                                            Uint64 dpnId, String rd, String interfaceName,
1526                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1527                                            TypedWriteTransaction<Operational> writeOperTx) {
1528         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1529                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1530         nhList.forEach((nh) -> {
1531             //IRT: remove routes from other vpns importing it
1532             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1533                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1534             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1535                 String vpnRd = vpn.getVrfId();
1536                 if (vpnRd != null) {
1537                     fibManager.removeOrUpdateFibEntry(vpnRd,
1538                             nextHop.getIpAddress(), nh, writeConfigTxn);
1539                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1540                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1541                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1542                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1543                 }
1544             }
1545         });
1546     }
1547
1548     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1549                                                  int lportTag, String gwMac, String gwIp, String interfaceName,
1550                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1551             throws ExecutionException, InterruptedException {
1552         final Uuid subnetId = nextHop.getSubnetId();
1553         if (nextHop.getSubnetGatewayMacAddress() == null) {
1554             // A valid mac-address was not available for this subnet-gateway-ip
1555             // So a connected-mac-address was used for this subnet and we need
1556             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1557             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1558         }
1559         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1560                 subnetId);
1561     }
1562
1563     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1564                                                   String interfaceName) {
1565         // This is either an extra-route (or) a learned IP via subnet-route
1566         List<String> nhList = null;
1567         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1568         if (nextHopIp == null || nextHopIp.isEmpty()) {
1569             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1570                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1571                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1572                     vpnName);
1573             nhList = emptyList();
1574         } else {
1575             nhList = Collections.singletonList(nextHopIp);
1576         }
1577         return nhList;
1578     }
1579
1580     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1581         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1582         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1583         // else use connected interface
1584         if (gwPort != null && gwPort.isSubnetIp()) {
1585             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1586                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1587             return Optional.of(gwPort.getMacAddress());
1588         }
1589         return Optional.absent();
1590     }
1591
1592     @Override
1593     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1594         final VpnInterface update) {
1595         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1596         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1597                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1598         if (original.equals(update)) {
1599             LOG.info("update: original {} update {} are same. No update required.", original, update);
1600             return;
1601         }
1602         final String vpnInterfaceName = update.getName();
1603         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1604         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1605         //handles switching between <internal VPN - external VPN>
1606         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1607             List<ListenableFuture<Void>> futures = new ArrayList<>();
1608             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1609                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1610                                 + "to newVpn(s) {}",
1611                         original.getName(), dpnId,
1612                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1613                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1614                 return emptyList();
1615             }
1616             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1617             return futures;
1618         });
1619     }
1620
1621     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1622                                                            VpnInterface original, VpnInterface update,
1623                                                            List<ListenableFuture<Void>> futures) {
1624         boolean isVpnInstanceUpdate = false;
1625         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1626         final String interfaceName = key.getName();
1627         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1628         List<String> oldVpnListCopy = new ArrayList<>();
1629         oldVpnListCopy.addAll(oldVpnList);
1630         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1631         List<String> newVpnListCopy = new ArrayList<>();
1632         newVpnListCopy.addAll(newVpnList);
1633
1634         oldVpnList.removeAll(newVpnList);
1635         newVpnList.removeAll(oldVpnListCopy);
1636         //This block will execute only on if there is a change in the VPN Instance.
1637         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1638             /*
1639              * Internet BGP-VPN Instance update with single router:
1640              * ====================================================
1641              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1642              *     1st VPN Instance : router VPN or external BGP-VPN.
1643              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1644              *
1645              * VPN Instance UPDATE:
1646              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1647              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1648              *
1649              * External BGP-VPN Instance update with single router:
1650              * ====================================================
1651              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1652              *
1653              * Updated VPN Instance will be always either internal router VPN to
1654              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1655              *
1656              * VPN Instance UPDATE:
1657              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1658              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1659              *
1660              * Dual Router VPN Instance Update:
1661              * ================================
1662              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1663              *
1664              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1665              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1666              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1667              *
1668              * Dual Router --> Associated with common external BGP-VPN Instance.
1669              * 1st router and 2nd router are getting associated with single External BGP-VPN
1670              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1671              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1672              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1673              *
1674              *
1675              */
1676             isVpnInstanceUpdate = true;
1677             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1678                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1679                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1680                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1681                     List<String> externalBgpVpnList = new ArrayList<>();
1682                     for (String newVpnName : newVpnListCopy) {
1683                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1684                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1685                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1686                                 .BgpvpnType.BGPVPNExternal) {
1687                             externalBgpVpnList.add(newVpnName);
1688                             break;
1689                         }
1690                     }
1691                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1692                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1693                             externalBgpVpnList, oldVpnListCopy, futures);
1694
1695                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1696                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1697                     //Identify the router VPN Instance and pass that value as oldVpnList
1698                     List<String> routerVpnList = new ArrayList<>();
1699                     for (String newVpnName : newVpnListCopy) {
1700                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1701                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1702                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1703                                 .BgpvpnType.VPN) {
1704                             routerVpnList.add(newVpnName);
1705                             break;
1706                         }
1707                     }
1708                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1709                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1710                             newVpnList, oldVpnListCopy, futures);
1711
1712                 } else {
1713                     //Handle remaining use cases.
1714                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1715                             oldVpnListCopy, futures);
1716                 }
1717             } else {
1718                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1719                         oldVpnListCopy, futures);
1720             }
1721         }
1722         return isVpnInstanceUpdate;
1723     }
1724
1725     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1726                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1727                                          List<String> newVpnList, List<String> oldVpnListCopy,
1728                                          List<ListenableFuture<Void>> futures) {
1729         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1730         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1731                 ? origAdjs.getAdjacency() : new ArrayList<>();
1732         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1733         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1734                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1735
1736         boolean isOldVpnRemoveCallExecuted = false;
1737         for (String oldVpnName : oldVpnList) {
1738             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1739                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1740             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1741             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1742                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1743                     newVpnList);
1744             isOldVpnRemoveCallExecuted = true;
1745         }
1746         //Wait for previous interface bindings to be removed
1747         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1748             try {
1749                 Thread.sleep(2000);
1750             } catch (InterruptedException e) {
1751                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1752             }
1753         }
1754         for (String newVpnName : newVpnList) {
1755             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1756             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1757                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1758                         + "onto vpnName {} ", interfaceName, newVpnName);
1759                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1760                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1761                                 + "from oldVpn(s) {} to newVpn {} ",
1762                         interfaceName, oldVpnListCopy, newVpnName);
1763                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1764                  * Use Case:
1765                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1766                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1767                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1768                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1769                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1770                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1771                  *     router VPN as well as Internet BGP-VPN.
1772                  *
1773                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1774                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1775                  *                                       update in existing router VPN instance.
1776                  */
1777                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1778                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1779                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1780                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1781                 }
1782             } else {
1783                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1784                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1785                         interfaceName, oldVpnListCopy, newVpnName);
1786             }
1787         }
1788     }
1789
1790     // TODO Clean up the exception handling
1791     @SuppressWarnings("checkstyle:IllegalCatch")
1792     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1793                                                                     String vpnInterfaceName,
1794                                                                     List<ListenableFuture<Void>> futures) {
1795         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1796         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1797                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1798         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1799         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1800                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1801
1802         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1803         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1804             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1805             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1806             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1807             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1808             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1809                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1810                 //set of prefix used as entry in prefix-to-interface datastore
1811                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1812                 // in bucket actions on bgp-vpn delete
1813                 Set<String> prefixListForRefreshFib = new HashSet<>();
1814                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1815                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1816                         operTx -> {
1817                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1818                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1819                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1820                                     update.getName(), newVpnName);
1821                             //handle both addition and removal of adjacencies
1822                             // currently, new adjacency may be an extra route
1823                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1824                             if (!oldAdjs.equals(newAdjs)) {
1825                                 for (Adjacency adj : copyNewAdjs) {
1826                                     if (copyOldAdjs.contains(adj)) {
1827                                         copyOldAdjs.remove(adj);
1828                                     } else {
1829                                         // add new adjacency
1830                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1831                                             try {
1832                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1833                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1834                                             } catch (RuntimeException e) {
1835                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1836                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1837                                             }
1838                                         }
1839                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1840                                             + " added to vpn interface {} on vpn {} dpnId {}",
1841                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1842                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1843                                     }
1844                                 }
1845                                 for (Adjacency adj : copyOldAdjs) {
1846                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1847                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1848                                             && !adj.isPhysNetworkFunc()) {
1849                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1850                                                 confTx);
1851                                             //remove FIB entry
1852                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1853                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1854                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1855                                             //remove BGP entry
1856                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1857                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1858                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1859                                             }
1860                                         } else {
1861                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1862                                                 operTx, confTx);
1863                                         }
1864                                     }
1865                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1866                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1867                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1868                                 }
1869                             }
1870                         })));
1871                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1872                     MoreExecutors.directExecutor());
1873                 futures.add(configTxFuture);
1874                 for (ListenableFuture<Void> future : futures) {
1875                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1876                             update.getName(), update.getVpnInstanceNames());
1877                 }
1878             } else {
1879                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1880                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1881             }
1882         }
1883         return futures;
1884     }
1885
1886     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1887         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1888                 .toString();
1889         // FIXME: separate this out somehow?
1890         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1891         lock.lock();
1892         try {
1893             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1894                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1895             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1896                 LogicalDatastoreType.OPERATIONAL, lriIid);
1897             if (opResult.isPresent()) {
1898                 LabelRouteInfo labelRouteInfo =
1899                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1900                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1901                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1902             }
1903             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1904                     nextHopIpList);
1905         } catch (ReadFailedException e) {
1906             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1907                     nextHopIpList);
1908         } catch (TransactionCommitFailedException e) {
1909             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1910                     nextHopIpList);
1911         } finally {
1912             lock.unlock();
1913         }
1914     }
1915
1916     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1917         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1918
1919         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1920         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1921                 .addAugmentation(SubnetRoute.class, route).build();
1922         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1923         InstanceIdentifierBuilder<VrfTables> idBuilder =
1924             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1925         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1926         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1927         if (writeConfigTxn != null) {
1928             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1929         } else {
1930             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1931         }
1932         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1933                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1934     }
1935
1936     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1937                                            Adjacency adj, Uint64 dpnId,
1938                                            TypedWriteTransaction<Operational> writeOperTxn,
1939                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1940                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1941                                            Set<String> prefixListForRefreshFib)
1942             throws ExecutionException, InterruptedException {
1943         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1944         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1945         try {
1946             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1947                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1948             if (optVpnInterface.isPresent()) {
1949                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1950                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1951                 String vpnName = currVpnIntf.getVpnInstanceName();
1952                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1953                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1954                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1955                         LogicalDatastoreType.OPERATIONAL, adjPath);
1956                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1957                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1958                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1959                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1960                 List<Adjacency> adjacencies = new ArrayList<>();
1961                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1962                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1963                 }
1964                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1965                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1966                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1967                 Adjacency operationalAdjacency = null;
1968                 //Handling dual stack neutron port primary adjacency
1969                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1970                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1971                             currVpnIntf.getName(), vpnName);
1972                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1973                             currVpnIntf.getName());
1974                     if (interfaceState != null) {
1975                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1976                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1977                             prefixListForRefreshFib);
1978                     }
1979                 }
1980                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1981                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1982                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1983                             : RouteOrigin.STATIC;
1984                     String nh = adj.getNextHopIpList().get(0);
1985                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1986                     // FIXME: separate out to somehow?
1987                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1988                     lock.lock();
1989                     try {
1990                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1991                                 vpnId, null, prefix, vpnName, nh, dpnId);
1992                         if (rdToAllocate.isPresent()) {
1993                             input.setRd(rdToAllocate.get());
1994                             operationalAdjacency = populator.createOperationalAdjacency(input);
1995                             int label = operationalAdjacency.getLabel().intValue();
1996                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1997                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1998                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1999                                     prefixListForRefreshFib, writeConfigTxn);
2000                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
2001                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
2002                                     vpnName, label, currVpnIntf.getName(), dpnId);
2003                         } else {
2004                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2005                                     vpnName, prefix);
2006                             return;
2007                         }
2008                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2009                         // Keeping the MPLS check for now.
2010                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2011                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2012                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2013                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
2014                             vpnsToImportRoute.forEach(vpn -> {
2015                                 if (vpn.getVrfId() != null) {
2016                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2017                                             vpnId, prefix,
2018                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2019                                             .ifPresent(
2020                                                 rds -> vpnManager.addExtraRoute(
2021                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2022                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2023                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2024                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2025                                 }
2026                             });
2027                         }
2028                     } finally {
2029                         lock.unlock();
2030                     }
2031                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2032                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2033                             currVpnIntf.getName(), vpnName);
2034
2035                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2036                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2037                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2038                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2039                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2040                             Prefixes.PrefixCue.PhysNetFunc);
2041                     if (vpnIntefaceConfig.isPresent()) {
2042                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2043                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2044                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2045                     }
2046
2047                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2048
2049                     writeOperTxn.merge(
2050                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2051                                     prefix), pnfPrefix, true);
2052
2053                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2054                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2055                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2056                             RouteOrigin.LOCAL, writeConfigTxn);
2057
2058                     input.setRd(adj.getVrfId());
2059                 }
2060                 if (operationalAdjacency == null) {
2061                     operationalAdjacency = populator.createOperationalAdjacency(input);
2062                 }
2063                 adjacencies.add(operationalAdjacency);
2064                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2065                 VpnInterfaceOpDataEntry newVpnIntf =
2066                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2067                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2068                                 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2069                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2070             }
2071         } catch (ReadFailedException e) {
2072             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2073                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2074         }
2075     }
2076
2077     @Nullable
2078     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2079         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2080         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2081     }
2082
2083     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2084                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2085                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2086         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2087         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2088         try {
2089             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2090                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2091             if (optVpnInterface.isPresent()) {
2092                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2093                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2094                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2095                         LogicalDatastoreType.OPERATIONAL, path);
2096                 if (optAdjacencies.isPresent()) {
2097                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2098
2099                     if (adjacencies != null && !adjacencies.isEmpty()) {
2100                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2101                         for (Adjacency adjacency : adjacencies) {
2102                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2103                                 String rd = adjacency.getVrfId();
2104                                 if (adj.getNextHopIpList() != null) {
2105                                     for (String nh : adj.getNextHopIpList()) {
2106                                         deleteExtraRouteFromCurrentAndImportingVpns(
2107                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2108                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2109                                     }
2110                                 } else if (adj.isPhysNetworkFunc()) {
2111                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2112                                             adj.getIpAddress(), adj.getSubnetId());
2113                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2114                                             null, writeConfigTxn);
2115                                 }
2116                                 break;
2117                             }
2118
2119                         }
2120                     }
2121                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2122                             dpnId, adj.getVrfId());
2123                 } else {
2124                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2125                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2126                 }
2127             }
2128         } catch (ReadFailedException e) {
2129             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2130                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2131         }
2132     }
2133
2134     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2135                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2136                                     TypedWriteTransaction<Operational> writeOperTx) {
2137         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2138                 destination, nextHop, vpnName, intfName, rd);
2139         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2140         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2141         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2142             String vpnRd = vpn.getVrfId();
2143             if (vpnRd != null) {
2144                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2145                         destination, nextHop, vpnName, intfName, vpnRd);
2146                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2147                         writeOperTx);
2148             }
2149         }
2150     }
2151
2152     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2153         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2154             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2155             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2156     }
2157
2158     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2159         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2160             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2161     }
2162
2163     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2164                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2165         if (vpnInterface == null) {
2166             return;
2167         }
2168         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2169         if (adjs == null) {
2170             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2171                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2172             return;
2173         }
2174         for (Adjacency adj : adjs) {
2175             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2176                 String primaryInterfaceIp = adj.getIpAddress();
2177                 String macAddress = adj.getMacAddress();
2178                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2179
2180                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2181                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2182                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
2183                     LOG.error(
2184                             "createFibEntryForRouterInterface: Unable to retrieve label for vpn pool {}, "
2185                                     + "vpninterface {}, vpn {}, rd {}",
2186                             VpnConstants.VPN_IDPOOL_NAME, interfaceName, vpnName, primaryRd);
2187                     return;
2188                 }
2189                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2190                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2191                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2192                         routerInt, label, writeConfigTxn);
2193                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2194                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2195                         macAddress);
2196             } else {
2197                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2198                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2199                         interfaceName, primaryRd, vpnName);
2200             }
2201         }
2202     }
2203
2204     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2205             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2206         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2207         String rd = vpnUtil.getVpnRd(vpnName);
2208         if (adjs != null) {
2209             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2210             for (Adjacency adj : adjsList) {
2211                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2212                     String primaryInterfaceIp = adj.getIpAddress();
2213                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2214                     fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2215                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2216                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2217                 }
2218             }
2219         } else {
2220             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2221                     vpnInterface.getName(), rd);
2222         }
2223     }
2224
2225     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2226         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2227         final String interfaceName = key.getName();
2228         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2229                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2230         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2231                   intefaceData.identifier, vpnName);
2232     }
2233
2234     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2235                                               VpnInterface vpnInterface, String vpnName) {
2236         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2237                .get(vpnName);
2238         if (vpnInterfaces == null) {
2239             vpnInterfaces = new ConcurrentLinkedQueue<>();
2240         }
2241         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2242         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2243         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2244                 vpnInterface.getName(), vpnName);
2245     }
2246
2247     public boolean isVpnInstanceReady(String vpnInstanceName) {
2248         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2249         if (vpnRd == null) {
2250             return false;
2251         }
2252         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2253
2254         return vpnInstanceOpDataEntry != null;
2255     }
2256
2257     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2258         // FIXME: separate out to somehow?
2259         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2260         lock.lock();
2261         try {
2262             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2263                     unprocessedVpnInterfaces.get(vpnInstanceName);
2264             if (vpnInterfaces != null) {
2265                 while (!vpnInterfaces.isEmpty()) {
2266                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2267                     if (hasVpnInstanceCreatedSuccessfully) {
2268                         processSavedInterface(savedInterface, vpnInstanceName);
2269                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2270                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2271                     } else {
2272                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2273                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2274                     }
2275                 }
2276             } else {
2277                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2278             }
2279         } finally {
2280             lock.unlock();
2281         }
2282     }
2283
2284     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2285             VpnInterface vpnInterface) {
2286         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2287         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2288         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2289         lock.lock();
2290         try {
2291             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2292                     unprocessedVpnInterfaces.get(firstVpnName);
2293             if (vpnInterfaces != null) {
2294                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2295                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2296                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2297                 }
2298             } else {
2299                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2300             }
2301         } finally {
2302             lock.unlock();
2303         }
2304     }
2305
2306     public void vpnInstanceIsReady(String vpnInstanceName) {
2307         processSavedInterfaces(vpnInstanceName, true);
2308     }
2309
2310     public void vpnInstanceFailed(String vpnInstanceName) {
2311         processSavedInterfaces(vpnInstanceName, false);
2312     }
2313
2314     private static class UnprocessedVpnInterfaceData {
2315         InstanceIdentifier<VpnInterface> identifier;
2316         VpnInterface vpnInterface;
2317
2318         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2319             this.identifier = identifier;
2320             this.vpnInterface = vpnInterface;
2321         }
2322
2323         @Override
2324         public int hashCode() {
2325             final int prime = 31;
2326             int result = 1;
2327             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2328             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2329             return result;
2330         }
2331
2332         @Override
2333         public boolean equals(Object obj) {
2334             if (this == obj) {
2335                 return true;
2336             }
2337             if (obj == null) {
2338                 return false;
2339             }
2340             if (getClass() != obj.getClass()) {
2341                 return false;
2342             }
2343             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2344             if (identifier == null) {
2345                 if (other.identifier != null) {
2346                     return false;
2347                 }
2348             } else if (!identifier.equals(other.identifier)) {
2349                 return false;
2350             }
2351             if (vpnInterface == null) {
2352                 if (other.vpnInterface != null) {
2353                     return false;
2354                 }
2355             } else if (!vpnInterface.equals(other.vpnInterface)) {
2356                 return false;
2357             }
2358             return true;
2359         }
2360     }
2361
2362     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2363         String primaryRd = vpnUtil.getVpnRd(vpnName);
2364         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2365         if (vpnInstanceOpData == null) {
2366             return;
2367         }
2368         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2369         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2370             return;
2371         }
2372         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2373         vpnToDpnLists.forEach(vpnToDpnList -> {
2374             if (vpnToDpnList.getVpnInterfaces() == null) {
2375                 return;
2376             }
2377             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2378                 try {
2379                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2380                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2381                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2382                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2383                     if (!vpnInterfaceOptional.isPresent()) {
2384                         return;
2385                     }
2386                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2387                             vpnInterface.getInterfaceName());
2388                     if (configVpnAdjacencies == null) {
2389                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2390                         return;
2391                     }
2392                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2393                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2394                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2395                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2396                     // in operational DS. These unprocessed adjacencies will be handled below.
2397                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2398                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2399                     configVpnAdjacencies.stream()
2400                         .filter(adjacency -> operationVpnAdjacencies.stream()
2401                                 .noneMatch(operationalAdjacency ->
2402                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2403                         .forEach(adjacency -> {
2404                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2405                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2406                                 () -> {
2407                                     // TODO Deal with sequencing — the config tx must only submitted
2408                                     // if the oper tx goes in
2409                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2410                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2411                                         futures.add(
2412                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2413                                                 //set of prefix used, as entry in prefix-to-interface datastore
2414                                                 // is prerequisite for refresh Fib to avoid race condition leading
2415                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2416                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2417                                                 ListenableFuture<Void> configTxFuture =
2418                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2419                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2420                                                             primaryRd, adjacency,
2421                                                             vpnInterfaceOptional.get().getDpnId(),
2422                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2423                                                 Futures.addCallback(configTxFuture,
2424                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2425                                                     MoreExecutors.directExecutor());
2426                                                 futures.add(configTxFuture);
2427                                             }));
2428                                         return futures;
2429                                     } else {
2430                                         return emptyList();
2431                                     }
2432                                 });
2433                         });
2434                 } catch (ReadFailedException e) {
2435                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2436                             vpnName, primaryRd);
2437                 }
2438             });
2439         });
2440     }
2441
2442     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2443         private final String interfaceName;
2444         private final boolean add;
2445         private final String txnDestination;
2446
2447         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2448             this.interfaceName = interfaceName;
2449             this.add = add;
2450             this.txnDestination = transactionDest;
2451         }
2452
2453         @Override
2454         public void onSuccess(Void voidObj) {
2455             if (add) {
2456                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2457                         interfaceName, txnDestination);
2458             } else {
2459                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2460             }
2461         }
2462
2463         @Override
2464         public void onFailure(Throwable throwable) {
2465             if (add) {
2466                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2467                         interfaceName, txnDestination, throwable);
2468             } else {
2469                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2470                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2471             }
2472         }
2473     }
2474
2475     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2476         private final String primaryRd;
2477         private final Set<String> prefixListForRefreshFib;
2478
2479         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2480             this.primaryRd = primaryRd;
2481             this.prefixListForRefreshFib = prefixListForRefreshFib;
2482         }
2483
2484         @Override
2485         public void onSuccess(Void voidObj) {
2486             prefixListForRefreshFib.forEach(prefix -> {
2487                 fibManager.refreshVrfEntry(primaryRd, prefix);
2488             });
2489         }
2490
2491         @Override
2492         public void onFailure(Throwable throwable) {
2493             LOG.debug("write Tx config operation failed", throwable);
2494         }
2495     }
2496 }